From 6c78541f15917412f01d0a36efca13a73b812958 Mon Sep 17 00:00:00 2001 From: Wil T Date: Mon, 12 Aug 2024 16:43:22 -0400 Subject: [PATCH 01/32] Use UV for package install and dependency resolution (#1016) * remove mac-specific action yml * rename test action yml * add mac testing to test action * use UV for faster lint+test action * consistent order of matrix variables in job names * use identical form for limited-dependencies TRUE * add pip intall verification * rename actions * fix name of pip install yml * fix syntax error in pip-install.yml * unique job name for pip-install * track lint/format pass/fail separately * re-order tests for faster failing * use ruff format --check flag --------- Co-authored-by: Shauna Gordon-McKeon --- .github/workflows/pip-install.yml | 60 ++++++++++++++++++++ .github/workflows/test-linux-windows.yml | 46 --------------- .github/workflows/test.yml | 72 ++++++++++++++++++++++++ .github/workflows/tests-mac.yml | 46 --------------- 4 files changed, 132 insertions(+), 92 deletions(-) create mode 100644 .github/workflows/pip-install.yml delete mode 100644 .github/workflows/test-linux-windows.yml create mode 100644 .github/workflows/test.yml delete mode 100644 .github/workflows/tests-mac.yml diff --git a/.github/workflows/pip-install.yml b/.github/workflows/pip-install.yml new file mode 100644 index 0000000000..99fe3c184a --- /dev/null +++ b/.github/workflows/pip-install.yml @@ -0,0 +1,60 @@ +name: Install with pip + +on: + pull_request: + paths: + - .github/workflows/pip-install.yml + - requirements.txt + - setup.py + push: + paths: + - .github/workflows/pip-install.yml + - requirements.txt + - setup.py + workflow_dispatch: + +env: + TESTING: 1 + +jobs: + pip-install: + strategy: + matrix: + os: + - ubuntu-latest + python-version: + - "3.12" + - "3.11" + - "3.10" + - "3.9" + - "3.8" + limited-dependencies: + - "" + - "TRUE" + include: + - os: macos-latest + python-version: "3.12" + - os: macos-latest + python-version: "3.12" + limited-dependencies: "TRUE" + - os: macos-latest + python-version: "3.8" + - os: macos-latest + python-version: "3.8" + limited-dependencies: "TRUE" + + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + - name: Install dependencies + env: + PARSONS_LIMITED_DEPENDENCIES: ${{ matrix.limited-dependencies }} + run: | + pip install -r requirements-dev.txt + pip install -e .[all] diff --git a/.github/workflows/test-linux-windows.yml b/.github/workflows/test-linux-windows.yml deleted file mode 100644 index ec191b7f8a..0000000000 --- a/.github/workflows/test-linux-windows.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: tests - -on: - pull_request: - branches: ["main", "major-release"] - push: - branches: ["main", "major-release"] - -env: - TESTING: 1 - -jobs: - build: - strategy: - matrix: - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] - limited-dependencies: ['','TRUE'] - os: [ubuntu-latest] # add in windows-latest to add windows testing - - runs-on: ${{ matrix.os }} - - steps: - - - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - cache: "pip" - - - name: Install dependencies - env: - PARSONS_LIMITED_DEPENDENCIES: ${{ matrix.limited-dependencies }} - run: | - python -m pip install -U pip - python -m pip install .[all] - python -m pip install -r requirements-dev.txt - - - name: Run tests - run: pytest -rf test/ - - - name: Check linting - run: | - ruff check parsons/ test/ useful_resources/ - ruff format --diff parsons/ test/ useful_resources/ diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000000..b8d06b9692 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,72 @@ +name: Pytest and ruff + +on: + pull_request: + paths: + - .github/workflows/test.yml + - requirements*.txt + - "**.py" + push: + paths: + - .github/workflows/test.yml + - requirements*.txt + - "**.py" + workflow_dispatch: + +env: + TESTING: 1 + +jobs: + test_and_lint: + strategy: + matrix: + os: + - ubuntu-latest + python-version: + - "3.12" + - "3.11" + - "3.10" + - "3.9" + - "3.8" + limited-dependencies: + - "" + - "TRUE" + include: + - os: macos-latest + python-version: "3.12" + - os: macos-latest + python-version: "3.12" + limited-dependencies: "TRUE" + - os: macos-latest + python-version: "3.8" + - os: macos-latest + python-version: "3.8" + limited-dependencies: "TRUE" + + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + - name: Install uv + run: | + pip install -U pip uv + - name: Install dependencies + env: + PARSONS_LIMITED_DEPENDENCIES: ${{ matrix.limited-dependencies }} + run: | + uv pip install --system -e .[all] + uv pip install --system -r requirements-dev.txt + - name: Lint + run: | + ruff check parsons/ test/ useful_resources/ + - name: Tests + run: pytest -rf test/ + - name: Format + run: | + ruff format --check --diff parsons/ test/ useful_resources/ + \ No newline at end of file diff --git a/.github/workflows/tests-mac.yml b/.github/workflows/tests-mac.yml deleted file mode 100644 index c895701836..0000000000 --- a/.github/workflows/tests-mac.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: tests for mac -# test mac on single python version as mac tests use 10x minutes/storage - -on: - pull_request: - branches: ["main", "major-release"] - push: - branches: ["main", "major-release"] - -env: - TESTING: 1 - -jobs: - build: - strategy: - matrix: - python-version: ['3.8', '3.11', '3.12'] - limited-dependencies: ['','TRUE'] - - runs-on: macos-latest - - steps: - - - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - cache: "pip" - - - name: Install dependencies - env: - PARSONS_LIMITED_DEPENDENCIES: ${{ matrix.limited-dependencies }} - run: | - python -m pip install -U pip - python -m pip install .[all] - python -m pip install -r requirements-dev.txt - - - name: Run tests - run: pytest - - - name: Check linting - run: | - ruff check parsons/ test/ useful_resources/ - ruff format --diff test/ useful_resources/ From 8757a550745aa77b91a249bfcfa40fe6003042ef Mon Sep 17 00:00:00 2001 From: Shauna Gordon-McKeon Date: Tue, 13 Aug 2024 16:59:12 -0400 Subject: [PATCH 02/32] Create security_scorecard.yml (#1116) * Create security_scorecard.yml * Update security_scorecard.yml add pull request as trigger so we can see the checks run in our PR * Update security_scorecard.yml add back the cron scheduler so it runs once a week --- .github/workflows/security_scorecard.yml | 73 ++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 .github/workflows/security_scorecard.yml diff --git a/.github/workflows/security_scorecard.yml b/.github/workflows/security_scorecard.yml new file mode 100644 index 0000000000..f6c9580595 --- /dev/null +++ b/.github/workflows/security_scorecard.yml @@ -0,0 +1,73 @@ +# This workflow uses actions that are not certified by GitHub. They are provided +# by a third-party and are governed by separate terms of service, privacy +# policy, and support documentation. + +name: Scorecard supply-chain security +on: + # For Branch-Protection check. Only the default branch is supported. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection + branch_protection_rule: + # To guarantee Maintained check is occasionally updated. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained + pull_request: + branches: [ "main" ] + schedule: + - cron: '45 16 * * 2' + +# Declare default permissions as read only. +permissions: read-all + +jobs: + analysis: + name: Scorecard analysis + runs-on: ubuntu-latest + permissions: + # Needed to upload the results to code-scanning dashboard. + security-events: write + # Needed to publish results and get a badge (see publish_results below). + id-token: write + # Uncomment the permissions below if installing in a private repository. + # contents: read + # actions: read + + steps: + - name: "Checkout code" + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + persist-credentials: false + + - name: "Run analysis" + uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1 + with: + results_file: results.sarif + results_format: sarif + # (Optional) "write" PAT token. Uncomment the `repo_token` line below if: + # - you want to enable the Branch-Protection check on a *public* repository, or + # - you are installing Scorecard on a *private* repository + # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action?tab=readme-ov-file#authentication-with-fine-grained-pat-optional. + # repo_token: ${{ secrets.SCORECARD_TOKEN }} + + # Public repositories: + # - Publish results to OpenSSF REST API for easy access by consumers + # - Allows the repository to include the Scorecard badge. + # - See https://github.com/ossf/scorecard-action#publishing-results. + # For private repositories: + # - `publish_results` will always be set to `false`, regardless + # of the value entered here. + publish_results: true + + # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF + # format to the repository Actions tab. + - name: "Upload artifact" + uses: actions/upload-artifact@97a0fba1372883ab732affbe8f94b823f91727db # v3.pre.node20 + with: + name: SARIF file + path: results.sarif + retention-days: 5 + + # Upload the results to GitHub's code scanning dashboard (optional). + # Commenting out will disable upload of results to your repo's Code Scanning dashboard + - name: "Upload to code-scanning" + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: results.sarif From 63dc85d4e6a82e56cfc13ebf1eae70f50c22b090 Mon Sep 17 00:00:00 2001 From: Austin Weisgrau <62900254+austinweisgrau@users.noreply.github.com> Date: Tue, 27 Aug 2024 10:44:10 -0700 Subject: [PATCH 03/32] SFTP utility can include timeout (#1081) --- parsons/sftp/sftp.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/parsons/sftp/sftp.py b/parsons/sftp/sftp.py index 60abd738a0..4f363aecd4 100644 --- a/parsons/sftp/sftp.py +++ b/parsons/sftp/sftp.py @@ -2,6 +2,7 @@ import re from contextlib import contextmanager from stat import S_ISDIR, S_ISREG +from typing import Optional import paramiko @@ -28,11 +29,21 @@ class SFTP(object): to authenticate stfp connection port: int Specify if different than the standard port 22 + timeout: int + Timeout argument for use when getting files through SFTP. `Returns:` SFTP Class """ - def __init__(self, host, username, password, port=22, rsa_private_key_file=None): + def __init__( + self, + host, + username, + password, + port=22, + rsa_private_key_file=None, + timeout: Optional[int] = None, + ): self.host = host if not self.host: raise ValueError("Missing the SFTP host name") @@ -47,6 +58,7 @@ def __init__(self, host, username, password, port=22, rsa_private_key_file=None) self.password = password self.rsa_private_key_file = rsa_private_key_file self.port = port + self.timeout = timeout @contextmanager def create_connection(self): @@ -63,8 +75,14 @@ def create_connection(self): # we need to read it in pkey = paramiko.RSAKey.from_private_key_file(self.rsa_private_key_file) - transport.connect(username=self.username, password=self.password, pkey=pkey) + transport.connect( + username=self.username, + password=self.password, + pkey=pkey, + ) conn = paramiko.SFTPClient.from_transport(transport) + if self.timeout: + conn.get_channel().settimeout(self.timeout) yield conn conn.close() transport.close() From c692a1cd8ea574557be26c388fea3745a21d3c75 Mon Sep 17 00:00:00 2001 From: Austin Weisgrau <62900254+austinweisgrau@users.noreply.github.com> Date: Tue, 27 Aug 2024 10:48:58 -0700 Subject: [PATCH 04/32] Pass custom delimiter from Table.from_csv to BigQuery load (#1083) If our source table is loaded from CSV with no transformations, the original source file will be directly loaded to GCS, we may need to pass along a custom delimiter to BigQuery. --- parsons/google/google_bigquery.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/parsons/google/google_bigquery.py b/parsons/google/google_bigquery.py index ab47cadce9..f8ba93e269 100644 --- a/parsons/google/google_bigquery.py +++ b/parsons/google/google_bigquery.py @@ -787,6 +787,15 @@ def copy( self._validate_copy_inputs(if_exists=if_exists, data_type=data_type) + # If our source table is loaded from CSV with no transformations + # The original source file will be directly loaded to GCS + # We may need to pass along a custom delimiter to BigQuery + # Otherwise we use the default comma + if isinstance(tbl.table, petl.io.csv_py3.CSVView): + csv_delimiter = tbl.table.csvargs.get("delimiter", ",") + else: + csv_delimiter = "," + job_config = self._process_job_config( job_config=job_config, destination_table_name=table_name, @@ -801,6 +810,7 @@ def copy( allow_jagged_rows=allow_jagged_rows, quote=quote, custom_schema=schema, + csv_delimiter=csv_delimiter, ) # Reorder schema to match table to ensure compatibility From 73754f18122452a885122aad7d7ce6a9615a0e0f Mon Sep 17 00:00:00 2001 From: Austin Weisgrau <62900254+austinweisgrau@users.noreply.github.com> Date: Tue, 27 Aug 2024 11:49:58 -0700 Subject: [PATCH 05/32] Enable GZIP on BigQuery extract to GCS (#1110) * Enable GZIP on BigQuery extract to GCS * Add docstring to extract method --- parsons/google/google_bigquery.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/parsons/google/google_bigquery.py b/parsons/google/google_bigquery.py index f8ba93e269..93e78a474a 100644 --- a/parsons/google/google_bigquery.py +++ b/parsons/google/google_bigquery.py @@ -1385,15 +1385,35 @@ def extract( gcs_bucket: str, gcs_blob_name: str, project: Optional[str] = None, + gzip: bool = False, ) -> None: + """ + Extracts a BigQuery table to a Google Cloud Storage bucket. + + Args: + dataset (str): The BigQuery dataset containing the table. + table_name (str): The name of the table to extract. + gcs_bucket (str): The GCS bucket where the table will be + exported. + gcs_blob_name (str): The name of the blob in the GCS + bucket. + project (Optional[str]): The Google Cloud project ID. If + not provided, the default project of the client is used. + gzip (bool): If True, the exported file will be compressed + using GZIP. Defaults to False. + """ + dataset_ref = bigquery.DatasetReference(project or self.client.project, dataset) table_ref = dataset_ref.table(table_name) gs_destination = f"gs://{gcs_bucket}/{gcs_blob_name}" - extract_job = self.client.extract_table( - table_ref, - gs_destination, - ) + if gzip: + job_config = bigquery.job.ExtractJobConfig() + job_config.compression = bigquery.Compression.GZIP + else: + job_config = None + + extract_job = self.client.extract_table(table_ref, gs_destination, job_config=job_config) extract_job.result() # Waits for job to complete. logger.info(f"Finished exporting query result to {gs_destination}.") From b49a98ca1da8dd64696b06efef413bb4516fb367 Mon Sep 17 00:00:00 2001 From: Paul Anzel Date: Fri, 6 Sep 2024 11:47:13 -0400 Subject: [PATCH 06/32] Step security changes (#1120) --- .github/workflows/pip-install.yml | 12 ++++++++++-- .github/workflows/test.yml | 13 ++++++++++--- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pip-install.yml b/.github/workflows/pip-install.yml index 99fe3c184a..66ed581be6 100644 --- a/.github/workflows/pip-install.yml +++ b/.github/workflows/pip-install.yml @@ -16,6 +16,9 @@ on: env: TESTING: 1 +permissions: # added using https://github.com/step-security/secure-repo + contents: read + jobs: pip-install: strategy: @@ -46,9 +49,14 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 + - name: Harden Runner + uses: step-security/harden-runner@5c7944e73c4c2a096b17a9cb74d65b6c2bbafbde # v2.9.1 + with: + egress-policy: audit + + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 with: python-version: ${{ matrix.python-version }} cache: pip diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b8d06b9692..be949bf63f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,6 +16,9 @@ on: env: TESTING: 1 +permissions: # added using https://github.com/step-security/secure-repo + contents: read + jobs: test_and_lint: strategy: @@ -46,9 +49,14 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 + - name: Harden Runner + uses: step-security/harden-runner@5c7944e73c4c2a096b17a9cb74d65b6c2bbafbde # v2.9.1 + with: + egress-policy: audit + + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 with: python-version: ${{ matrix.python-version }} cache: pip @@ -69,4 +77,3 @@ jobs: - name: Format run: | ruff format --check --diff parsons/ test/ useful_resources/ - \ No newline at end of file From 81f622c8fbcaa94b87269a6d784c09e4a7ffb82e Mon Sep 17 00:00:00 2001 From: Wil T Date: Fri, 6 Sep 2024 12:18:34 -0400 Subject: [PATCH 07/32] Implement Windows testing (#1119) * add windows tests to test action * test pip install on windows * update us package for 3.12 support * skip failing test on windows --------- Co-authored-by: Shauna Gordon-McKeon --- .github/workflows/pip-install.yml | 1 + .github/workflows/test.yml | 2 ++ requirements.txt | 2 +- test/test_targetsmart/test_targetsmart_smartmatch.py | 3 +++ 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pip-install.yml b/.github/workflows/pip-install.yml index 66ed581be6..216a99d77a 100644 --- a/.github/workflows/pip-install.yml +++ b/.github/workflows/pip-install.yml @@ -25,6 +25,7 @@ jobs: matrix: os: - ubuntu-latest + - windows-latest python-version: - "3.12" - "3.11" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index be949bf63f..97598adcc9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,6 +25,7 @@ jobs: matrix: os: - ubuntu-latest + - windows-latest python-version: - "3.12" - "3.11" @@ -77,3 +78,4 @@ jobs: - name: Format run: | ruff format --check --diff parsons/ test/ useful_resources/ + diff --git a/requirements.txt b/requirements.txt index 262b9f9580..b801d88b6c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -45,6 +45,6 @@ xmltodict==0.11.0 # TODO Remove when we have a TMC-specific Docker image jinja2>=3.0.2 selenium==3.141.0 -us==3.1.1 +us==3.2.0 sshtunnel==0.4.0 diff --git a/test/test_targetsmart/test_targetsmart_smartmatch.py b/test/test_targetsmart/test_targetsmart_smartmatch.py index f39c56fd7c..5bff6deb84 100644 --- a/test/test_targetsmart/test_targetsmart_smartmatch.py +++ b/test/test_targetsmart/test_targetsmart_smartmatch.py @@ -1,9 +1,11 @@ import csv import io import gzip +import sys import petl import pytest + from parsons.targetsmart.targetsmart_api import TargetSmartAPI @@ -70,6 +72,7 @@ def submit_filename(): return "parsons_test.csv" +@pytest.mark.skipif(sys.platform == "win32", reason="need to fix this test on windows") def test_smartmatch( intable, submit_filename, From 30be14b9d96c0b0b7b4fa251099eceb5e64b3439 Mon Sep 17 00:00:00 2001 From: Wil T Date: Fri, 6 Sep 2024 12:25:22 -0400 Subject: [PATCH 08/32] circleci updates (#1122) * use uv in circleci * use python 3.12 in circleci --- .circleci/config.yml | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 69b7a43726..682c8b0ddb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -20,46 +20,47 @@ jobs: # Documentation CI docs-build: docker: - - image: cimg/python:3.10 + - image: cimg/python:3.12 steps: - checkout # Download and cache dependencies - restore_cache: keys: - - v2-dependencies-python3.10-{{ checksum "./docs/requirements.txt" }} + - v2-dependencies-python3.12-{{ checksum "./docs/requirements.txt" }} # fallback to using the latest cache if no exact match is found - - v2-dependencies-python3.10- + - v2-dependencies-python3.12- - run: name: Install dependencies # Note that we the circleci node image installs stuff with a user "circleci", rather # than root. So we need to tell npm where to install stuff. command: | - python3 -m venv venv - . venv/bin/activate - pip install -r ./docs/requirements.txt + pip install uv + uv venv + source .venv/bin/activate + uv pip install -r ./docs/requirements.txt - save_cache: paths: - ./venv - key: v2-dependencies-python3.10-{{ checksum "./docs/requirements.txt" }} + key: v2-dependencies-python3.12-{{ checksum "./docs/requirements.txt" }} - run: name: Build docs command: | - . venv/bin/activate + source .venv/bin/activate cd docs/ make deploy_docs cd .. docs-build-deploy: docker: - - image: cimg/python:3.10-node + - image: cimg/python:3.12-node steps: - checkout # Download and cache dependencies - restore_cache: keys: - - v2-dependencies-python3.10-{{ checksum "./docs/requirements.txt" }} + - v2-dependencies-python3.12-{{ checksum "./docs/requirements.txt" }} # fallback to using the latest cache if no exact match is found - - v2-dependencies-python3.10- + - v2-dependencies-python3.12- - run: name: Install dependencies # Note that we the circleci node image installs stuff with a user "circleci", rather @@ -67,13 +68,14 @@ jobs: command: | npm set prefix=/home/circleci/npm npm install -g --silent gh-pages@2.0.1 - python3 -m venv venv - . venv/bin/activate - pip install -r ./docs/requirements.txt + pip install uv + uv venv + source .venv/bin/activate + uv pip install -r ./docs/requirements.txt - save_cache: paths: - ./venv - key: v2-dependencies-python3.10-{{ checksum "./docs/requirements.txt" }} + key: v2-dependencies-python3.12-{{ checksum "./docs/requirements.txt" }} - add_ssh_keys: # This SSH key is "CircleCI Docs" in https://github.com/move-coop/parsons/settings/keys # We need write access to the Parsons repo, so we can push the "gh-pages" branch. @@ -85,7 +87,7 @@ jobs: # (This file tell Github Pages that we want to include all files in docs/, including those # that start with an underscore like _static/). command: | - . venv/bin/activate + source .venv/bin/activate cd docs/ make deploy_docs cd .. From 009647cfcf6fd29b7daf766ea93bd584a3c9b5d8 Mon Sep 17 00:00:00 2001 From: Paul Anzel Date: Fri, 6 Sep 2024 12:38:56 -0400 Subject: [PATCH 09/32] Version bump for certain packages (#1121) * Version bump * Suds version fix * Higher mysql version * Pin google-api-core * Pin curlify, don't upgrade mysql --- requirements.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index b801d88b6c..dbdb158d4d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,9 +6,11 @@ braintree==4.17.1 bs4==0.0.1 censusgeocode==0.4.3.post1 civis==1.16.1 +curlify==2.2.1 dbt_redshift==1.4.0 docutils<0.18,>=0.14 facebook-business==13.0.0 +google-api-core==2.19.2 google-api-python-client==1.7.7 google-auth==2.29.0 google-cloud-bigquery==3.23.1 @@ -27,14 +29,14 @@ petl==1.7.15 psycopg2-binary==2.9.9 PyGitHub==1.51 python-dateutil==2.8.2 -requests==2.31.0 +requests==2.32.0 requests_oauthlib==1.3.0 setuptools==70.0.0 simple-salesforce==1.11.6 simplejson==3.16.0 slackclient==1.3.0 sqlalchemy >= 1.4.22, != 1.4.33, < 2.0.0 # Prefect does not work with 1.4.33 and >3.0.0 has breaking changes -suds-py3==1.3.4.0 +suds-py3==1.4.4.1 surveygizmo==1.2.3 twilio==8.2.1 urllib3==1.26.19 From 9a61b7cdb62dbb5ee59cfae45282a7238583cfbb Mon Sep 17 00:00:00 2001 From: Wil T Date: Fri, 6 Sep 2024 12:49:39 -0400 Subject: [PATCH 10/32] Update test.yml (#1117) --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 97598adcc9..176fa5245d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -72,7 +72,7 @@ jobs: uv pip install --system -r requirements-dev.txt - name: Lint run: | - ruff check parsons/ test/ useful_resources/ + ruff check --output-format=github parsons/ test/ useful_resources/ - name: Tests run: pytest -rf test/ - name: Format From 948b08bc2a907b0837f5ecd98638a9023059f9df Mon Sep 17 00:00:00 2001 From: retacg <62267774+retacg@users.noreply.github.com> Date: Tue, 10 Sep 2024 16:30:34 -0600 Subject: [PATCH 11/32] Add skipMatching to apply_response() (#1134) * Add skipMatching * adding comma --- parsons/ngpvan/people.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/parsons/ngpvan/people.py b/parsons/ngpvan/people.py index 7fd84c455b..006e2d555f 100644 --- a/parsons/ngpvan/people.py +++ b/parsons/ngpvan/people.py @@ -587,6 +587,7 @@ def apply_response( omit_contact=False, phone=None, campaignId=None, + skip_matching=False, ): """ Apply responses such as survey questions, activist codes, and volunteer actions @@ -624,6 +625,8 @@ def apply_response( `Optional`; Phone number of any type (Work, Cell, Home) campaignId: int `Optional`; a valid Campaign ID. + skip_matching: boolean + `Optional`; if set to true, skips matching/de-duping of contact history. Defaults to a null value, aka false. `Returns:` ``True`` if successful @@ -652,6 +655,7 @@ def apply_response( "dateCanvassed": date_canvassed, "omitActivistCodeContactHistory": omit_contact, "campaignId": campaignId, + "skipMatching": skip_matching, }, "resultCodeId": result_code_id, } From 1163cc025a03c5da0f4c2cda52134f1eee5f40cf Mon Sep 17 00:00:00 2001 From: Wil T Date: Tue, 17 Sep 2024 11:56:33 -0400 Subject: [PATCH 12/32] Update Dockerfile with uv (#1135) --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 895da28333..6cc7d00a3b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,7 +29,8 @@ ENV DISPLAY=:99 RUN mkdir /src COPY requirements.txt /src/ -RUN pip install -r /src/requirements.txt +RUN pip install uv +RUN uv pip install --system -r /src/requirements.txt COPY . /src/ WORKDIR /src From 710532aa5db4f24bac73cc6ff04c458b6670ecc9 Mon Sep 17 00:00:00 2001 From: lisamessier <97008179+lisamessier@users.noreply.github.com> Date: Thu, 19 Sep 2024 14:21:00 -0400 Subject: [PATCH 13/32] updated README with correct instructions for installing from Github (#1141) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bd8540009e..28aa23a716 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ You can install the most recent release by running: `pip install parsons[all]` ### Install from Github To access the most recent code base that may contain features not yet included in the latest release, download this -repository and then run `python setup.py develop`. +repository and then run `python -m pip install git+https://github.com/move-coop/parsons.git`. ### Docker Container From 22c31758b756721337a0fa61fc76f569c4e6c346 Mon Sep 17 00:00:00 2001 From: Austin Weisgrau <62900254+austinweisgrau@users.noreply.github.com> Date: Thu, 10 Oct 2024 08:10:03 -0700 Subject: [PATCH 14/32] email parse depends on python patch version (#1146) --- test/test_gmail/test_gmail.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/test/test_gmail/test_gmail.py b/test/test_gmail/test_gmail.py index 35fc992774..d31d488b33 100644 --- a/test/test_gmail/test_gmail.py +++ b/test/test_gmail/test_gmail.py @@ -2,10 +2,10 @@ import json import os import requests_mock +import email import unittest import shutil import base64 -import email _dir = os.path.dirname(__file__) @@ -516,10 +516,27 @@ def test__validate_email_string(self): {"email": "", "expected": True}, {"email": "Sender sender@email.com", "expected": False}, {"email": "Sender ", "expected": False}, - {"email": "Sender ", "expected": True}, - {"email": "Sender ", "expected": True}, ] + # The behavior of email.parseaddr depends on the python patch version + # See https://github.com/python/cpython/issues/102988 + # or associated changelogs, e.g. + # https://docs.python.org/3.8/whatsnew/changelog.html#python-3-8-20-final + if getattr(email.utils, "supports_strict_parsing", False): + emails.extend( + [ + {"email": "Sender ", "expected": False}, + {"email": "Sender ", "expected": False}, + ] + ) + else: + emails.extend( + [ + {"email": "Sender ", "expected": True}, + {"email": "Sender ", "expected": True}, + ] + ) + for e in emails: if e["expected"]: self.assertTrue(self.gmail._validate_email_string(e["email"])) From 1642d6e84bd36c20908ed2918f89a1393bd29d81 Mon Sep 17 00:00:00 2001 From: Austin Weisgrau <62900254+austinweisgrau@users.noreply.github.com> Date: Thu, 10 Oct 2024 08:32:55 -0700 Subject: [PATCH 15/32] BigQuery copy method can convert dict column to JSON string (#1143) By default without this change, a dict column will cause the copy method to fail. --- parsons/google/google_bigquery.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/parsons/google/google_bigquery.py b/parsons/google/google_bigquery.py index 93e78a474a..49c17ed23a 100644 --- a/parsons/google/google_bigquery.py +++ b/parsons/google/google_bigquery.py @@ -1,5 +1,6 @@ import datetime import logging +import json import pickle import random import uuid @@ -745,6 +746,7 @@ def copy( allow_jagged_rows: bool = True, quote: Optional[str] = None, schema: Optional[List[dict]] = None, + convert_dict_columns_to_json: bool = True, **load_kwargs, ): """ @@ -774,6 +776,8 @@ def copy( template_table: str Table name to be used as the load schema. Load operation wil use the same columns and data types as the template table. + convert_dict_columns_to_json: bool + If set to True, will convert any dict columns (which cannot by default be successfully loaded to BigQuery to JSON strings) **load_kwargs: kwargs Arguments to pass to the underlying load_table_from_uri call on the BigQuery client. @@ -796,6 +800,19 @@ def copy( else: csv_delimiter = "," + if convert_dict_columns_to_json: + # Convert dict columns to JSON strings + for field in tbl.get_columns_type_stats(): + if "dict" in field["type"]: + new_petl = tbl.table.addfield( + field["name"] + "_replace", lambda row: json.dumps(row[field["name"]]) + ) + new_tbl = Table(new_petl) + new_tbl.remove_column(field["name"]) + new_tbl.rename_column(field["name"] + "_replace", field["name"]) + new_tbl.materialize() + tbl = new_tbl + job_config = self._process_job_config( job_config=job_config, destination_table_name=table_name, From 5aaf2451963e553cc0c5bcf404ecb6687797a395 Mon Sep 17 00:00:00 2001 From: Paul Anzel Date: Thu, 10 Oct 2024 11:09:30 -0500 Subject: [PATCH 16/32] New requests (#1130) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index dbdb158d4d..bafa063f1d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,7 +29,7 @@ petl==1.7.15 psycopg2-binary==2.9.9 PyGitHub==1.51 python-dateutil==2.8.2 -requests==2.32.0 +requests==2.32.3 requests_oauthlib==1.3.0 setuptools==70.0.0 simple-salesforce==1.11.6 From 1d1f9882a4215d8f06968ca148497c42052b3884 Mon Sep 17 00:00:00 2001 From: Wil T Date: Thu, 10 Oct 2024 15:50:08 -0400 Subject: [PATCH 17/32] Github Actions Rebuild (#1132) * update ruff * add dependabot * add dependency review * harden security scorecard * run scorecard on major-release PRs * use defusedxml * add usedforsecurity flag to hashlib.md5 * correct comment for sqlalchemy * modularize python check action * update actions/upload-artifact --- .github/dependabot.yml | 21 +++ .github/workflows/dependency-review.yml | 28 +++ .github/workflows/pip-install.yml | 69 ------- .github/workflows/python-checks.yml | 168 ++++++++++++++++++ .github/workflows/security_scorecard.yml | 14 +- .github/workflows/test.yml | 81 --------- .pre-commit-config.yaml | 3 +- parsons/etl/etl.py | 35 +++- parsons/scytl/scytl.py | 2 +- parsons/targetsmart/targetsmart_automation.py | 4 +- pyproject.toml | 18 ++ requirements-dev.txt | 14 +- requirements.txt | 3 +- setup.py | 3 +- 14 files changed, 288 insertions(+), 175 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/dependency-review.yml delete mode 100644 .github/workflows/pip-install.yml create mode 100644 .github/workflows/python-checks.yml delete mode 100644 .github/workflows/test.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..86357362a1 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,21 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + + - package-ecosystem: docker + directory: / + schedule: + interval: weekly + + - package-ecosystem: pip + directory: /docs + schedule: + interval: weekly + + - package-ecosystem: pip + directory: / + schedule: + interval: weekly \ No newline at end of file diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml new file mode 100644 index 0000000000..3cfec7b486 --- /dev/null +++ b/.github/workflows/dependency-review.yml @@ -0,0 +1,28 @@ +# Dependency Review Action +# +# This Action will scan dependency manifest files that change as part of a Pull Request, +# surfacing known-vulnerable versions of the packages declared or updated in the PR. +# Once installed, if the workflow run is marked as required, +# PRs introducing known-vulnerable packages will be blocked from merging. +# +# Source repository: https://github.com/actions/dependency-review-action +name: Dependency review + +on: [pull_request] + +permissions: + contents: read + +jobs: + dependency-review: + runs-on: ubuntu-latest + steps: + - name: Harden Runner + uses: step-security/harden-runner@5c7944e73c4c2a096b17a9cb74d65b6c2bbafbde # v2.9.1 + with: + egress-policy: audit + + - name: 'Checkout Repository' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - name: 'Dependency Review' + uses: actions/dependency-review-action@5a2ce3f5b92ee19cbb1541a4984c76d921601d7c # v4.3.4 \ No newline at end of file diff --git a/.github/workflows/pip-install.yml b/.github/workflows/pip-install.yml deleted file mode 100644 index 216a99d77a..0000000000 --- a/.github/workflows/pip-install.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: Install with pip - -on: - pull_request: - paths: - - .github/workflows/pip-install.yml - - requirements.txt - - setup.py - push: - paths: - - .github/workflows/pip-install.yml - - requirements.txt - - setup.py - workflow_dispatch: - -env: - TESTING: 1 - -permissions: # added using https://github.com/step-security/secure-repo - contents: read - -jobs: - pip-install: - strategy: - matrix: - os: - - ubuntu-latest - - windows-latest - python-version: - - "3.12" - - "3.11" - - "3.10" - - "3.9" - - "3.8" - limited-dependencies: - - "" - - "TRUE" - include: - - os: macos-latest - python-version: "3.12" - - os: macos-latest - python-version: "3.12" - limited-dependencies: "TRUE" - - os: macos-latest - python-version: "3.8" - - os: macos-latest - python-version: "3.8" - limited-dependencies: "TRUE" - - runs-on: ${{ matrix.os }} - - steps: - - name: Harden Runner - uses: step-security/harden-runner@5c7944e73c4c2a096b17a9cb74d65b6c2bbafbde # v2.9.1 - with: - egress-policy: audit - - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 - with: - python-version: ${{ matrix.python-version }} - cache: pip - - name: Install dependencies - env: - PARSONS_LIMITED_DEPENDENCIES: ${{ matrix.limited-dependencies }} - run: | - pip install -r requirements-dev.txt - pip install -e .[all] diff --git a/.github/workflows/python-checks.yml b/.github/workflows/python-checks.yml new file mode 100644 index 0000000000..dfbb5c3a14 --- /dev/null +++ b/.github/workflows/python-checks.yml @@ -0,0 +1,168 @@ +name: Python checks + +on: + push: + branches: [ "main", "major-release" ] + pull_request: + branches: [ "main", "major-release" ] + workflow_dispatch: + +permissions: + contents: read + +jobs: + test: + strategy: + fail-fast: false + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + os: ["ubuntu-latest", "windows-latest", "macos-latest"] + limited-dependencies: ["", "TRUE"] + + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Install uv + uses: install-pinned/uv@de03c60d508703a83d3f8f49afcf1249590ecda1 # 0.4.12 + + - name: Install dependencies + env: + PARSONS_LIMITED_DEPENDENCIES: ${{ matrix.limited-dependencies }} + run: | + uv pip install --system -e .[all] + uv pip install --system -r requirements-dev.txt + + - name: Test with pytest + run: | + pytest + + ruff-format: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Set up Python 3.12 + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 + with: + python-version: "3.12" + + - name: Install uv + uses: install-pinned/uv@de03c60d508703a83d3f8f49afcf1249590ecda1 # 0.4.12 + + - name: Install dependencies + run: | + uv pip install --system -r requirements-dev.txt + + - name: Run ruff format + run: | + ruff format --diff --target-version=py38 . + + ruff: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Set up Python 3.12 + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 + with: + python-version: "3.12" + + - name: Install uv + uses: install-pinned/uv@de03c60d508703a83d3f8f49afcf1249590ecda1 # 0.4.12 + + - name: Install dependencies + run: | + uv pip install --system -r requirements-dev.txt + + - name: Run ruff + run: | + ruff check --output-format=github . + + bandit: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Set up Python 3.12 + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 + with: + python-version: "3.12" + + - name: Install uv + uses: install-pinned/uv@de03c60d508703a83d3f8f49afcf1249590ecda1 # 0.4.12 + + - name: Install bandit + run: | + uv pip install --system -r requirements-dev.txt + + - name: Run bandit scan + run: | + bandit -c pyproject.toml -r . -ll -ii + + coverage: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Set up Python 3.12 + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 + with: + python-version: "3.12" + + - name: Install uv + uses: install-pinned/uv@de03c60d508703a83d3f8f49afcf1249590ecda1 # 0.4.12 + + - name: Install dependencies + run: | + uv pip install --system -e .[all] + uv pip install --system -r requirements-dev.txt + + - name: Test with pytest + run: | + coverage run -m pytest + + - name: Check coverage + run: | + coverage report -m --skip-covered --fail-under=75 + + pip-install: + strategy: + fail-fast: false + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + os: ["ubuntu-latest", "windows-latest", "macos-latest"] + limited-dependencies: ["", "TRUE"] + + runs-on: ${{ matrix.os }} + + steps: + - name: Harden Runner + uses: step-security/harden-runner@5c7944e73c4c2a096b17a9cb74d65b6c2bbafbde # v2.9.1 + with: + egress-policy: audit + + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 + with: + python-version: ${{ matrix.python-version }} + cache: pip + + - name: Install dependencies + env: + PARSONS_LIMITED_DEPENDENCIES: ${{ matrix.limited-dependencies }} + run: | + pip install -r requirements-dev.txt + pip install -e .[all] diff --git a/.github/workflows/security_scorecard.yml b/.github/workflows/security_scorecard.yml index f6c9580595..3519c61bd0 100644 --- a/.github/workflows/security_scorecard.yml +++ b/.github/workflows/security_scorecard.yml @@ -3,6 +3,7 @@ # policy, and support documentation. name: Scorecard supply-chain security + on: # For Branch-Protection check. Only the default branch is supported. See # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection @@ -10,7 +11,7 @@ on: # To guarantee Maintained check is occasionally updated. See # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained pull_request: - branches: [ "main" ] + branches: [ "main", "major-release" ] schedule: - cron: '45 16 * * 2' @@ -31,6 +32,11 @@ jobs: # actions: read steps: + - name: Harden Runner + uses: step-security/harden-runner@5c7944e73c4c2a096b17a9cb74d65b6c2bbafbde # v2.9.1 + with: + egress-policy: audit + - name: "Checkout code" uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: @@ -59,7 +65,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" - uses: actions/upload-artifact@97a0fba1372883ab732affbe8f94b823f91727db # v3.pre.node20 + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: name: SARIF file path: results.sarif @@ -68,6 +74,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard (optional). # Commenting out will disable upload of results to your repo's Code Scanning dashboard - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@v3 + uses: github/codeql-action/upload-sarif@4dd16135b69a43b6c8efb853346f8437d92d3c93 # v3.26.6 with: - sarif_file: results.sarif + sarif_file: results.sarif \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 176fa5245d..0000000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,81 +0,0 @@ -name: Pytest and ruff - -on: - pull_request: - paths: - - .github/workflows/test.yml - - requirements*.txt - - "**.py" - push: - paths: - - .github/workflows/test.yml - - requirements*.txt - - "**.py" - workflow_dispatch: - -env: - TESTING: 1 - -permissions: # added using https://github.com/step-security/secure-repo - contents: read - -jobs: - test_and_lint: - strategy: - matrix: - os: - - ubuntu-latest - - windows-latest - python-version: - - "3.12" - - "3.11" - - "3.10" - - "3.9" - - "3.8" - limited-dependencies: - - "" - - "TRUE" - include: - - os: macos-latest - python-version: "3.12" - - os: macos-latest - python-version: "3.12" - limited-dependencies: "TRUE" - - os: macos-latest - python-version: "3.8" - - os: macos-latest - python-version: "3.8" - limited-dependencies: "TRUE" - - runs-on: ${{ matrix.os }} - - steps: - - name: Harden Runner - uses: step-security/harden-runner@5c7944e73c4c2a096b17a9cb74d65b6c2bbafbde # v2.9.1 - with: - egress-policy: audit - - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 - with: - python-version: ${{ matrix.python-version }} - cache: pip - - name: Install uv - run: | - pip install -U pip uv - - name: Install dependencies - env: - PARSONS_LIMITED_DEPENDENCIES: ${{ matrix.limited-dependencies }} - run: | - uv pip install --system -e .[all] - uv pip install --system -r requirements-dev.txt - - name: Lint - run: | - ruff check --output-format=github parsons/ test/ useful_resources/ - - name: Tests - run: pytest -rf test/ - - name: Format - run: | - ruff format --check --diff parsons/ test/ useful_resources/ - diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8b09c6e831..895355120f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - # Ruff version. - rev: v0.5.3 + rev: v0.6.9 hooks: # Run the linter. - id: ruff diff --git a/parsons/etl/etl.py b/parsons/etl/etl.py index 25c696f192..84563a4a88 100644 --- a/parsons/etl/etl.py +++ b/parsons/etl/etl.py @@ -1,4 +1,5 @@ import logging +import sys import petl @@ -658,10 +659,20 @@ def unpack_nested_columns_as_rows(self, column, key="id", expand_original=False) orig.concat(melted_list) # Add unique id column by hashing all the other fields if "uid" not in self.columns: - orig.add_column( - "uid", - lambda row: hashlib.md5(str.encode("".join([str(x) for x in row]))).hexdigest(), - ) + if sys.version_info.minor >= 9: + orig.add_column( + "uid", + lambda row: hashlib.md5( + str.encode("".join([str(x) for x in row])), usedforsecurity=False + ).hexdigest(), + ) + elif sys.version_info.minor < 9: + orig.add_column( + "uid", + lambda row: hashlib.md5( # nosec B324 + str.encode("".join([str(x) for x in row])) + ).hexdigest(), + ) orig.move_column("uid", 0) # Rename value column in case this is done again to this Table @@ -673,10 +684,18 @@ def unpack_nested_columns_as_rows(self, column, key="id", expand_original=False) else: orig = self.remove_column(column) # Add unique id column by hashing all the other fields - melted_list.add_column( - "uid", - lambda row: hashlib.md5(str.encode("".join([str(x) for x in row]))).hexdigest(), - ) + if sys.version_info.minor >= 9: + melted_list.add_column( + "uid", + lambda row: hashlib.md5( + str.encode("".join([str(x) for x in row])), usedforsecurity=False + ).hexdigest(), + ) + elif sys.version_info.minor < 9: + melted_list.add_column( + "uid", + lambda row: hashlib.md5(str.encode("".join([str(x) for x in row]))).hexdigest(), # nosec B324 + ) melted_list.move_column("uid", 0) output = melted_list diff --git a/parsons/scytl/scytl.py b/parsons/scytl/scytl.py index 0df28ba079..cbd8f1dd4c 100644 --- a/parsons/scytl/scytl.py +++ b/parsons/scytl/scytl.py @@ -1,7 +1,7 @@ import zipfile import csv import requests -import xml.etree.ElementTree as ET +import defusedxml.ElementTree as ET import typing as t from datetime import datetime from dateutil.parser import parse as parsedate diff --git a/parsons/targetsmart/targetsmart_automation.py b/parsons/targetsmart/targetsmart_automation.py index b23273cd97..2673c29c96 100644 --- a/parsons/targetsmart/targetsmart_automation.py +++ b/parsons/targetsmart/targetsmart_automation.py @@ -25,12 +25,12 @@ from parsons.etl.table import Table from parsons.utilities.files import create_temp_file from parsons.utilities import check_env -import xml.etree.ElementTree as ET import uuid import time import logging -import xmltodict +import defusedxml.ElementTree as ET +import xmltodict TS_STFP_HOST = "transfer.targetsmart.com" TS_SFTP_PORT = 22 diff --git a/pyproject.toml b/pyproject.toml index feca19eff1..e51bdeb3ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,10 @@ [tool.ruff] # Exclude a variety of commonly ignored directories. +include = [ + "parsons/**/*.py", + "test/**/*.py", + "useful_resources/**/*.py" +] exclude = [ ".bzr", ".direnv", @@ -83,4 +88,17 @@ testpaths = [ filterwarnings = [ # Warnings triggered by libraries we use (not our own code) "ignore:invalid escape sequence:DeprecationWarning" +] + +[tool.bandit] +exclude_dirs = [ + ".venv/" +] + +[tool.isort] +profile = "hug" +src_paths = [ + "parsons/", + "test/", + "useful_resources/" ] \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt index 5702ed94dc..20a822bfc6 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,8 +1,10 @@ # Testing Requirements -pycodestyle==2.11.1 +bandit[toml]==1.7.10 +coverage==7.6.1 +pycodestyle==2.12.1 pytest-datadir==1.5.0 -pytest-mock==3.12.0 -pytest==8.1.1 -requests-mock==1.11.0 -ruff==0.5.5 -testfixtures==8.1.0 \ No newline at end of file +pytest-mock==3.14.0 +pytest==8.3.3 +requests-mock==1.12.1 +ruff==0.6.9 +testfixtures==8.3.0 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index bafa063f1d..5b749fb8f6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,7 @@ civis==1.16.1 curlify==2.2.1 dbt_redshift==1.4.0 docutils<0.18,>=0.14 +defusedxml>=0.7.1, <=0.8.0 facebook-business==13.0.0 google-api-core==2.19.2 google-api-python-client==1.7.7 @@ -35,7 +36,7 @@ setuptools==70.0.0 simple-salesforce==1.11.6 simplejson==3.16.0 slackclient==1.3.0 -sqlalchemy >= 1.4.22, != 1.4.33, < 2.0.0 # Prefect does not work with 1.4.33 and >3.0.0 has breaking changes +sqlalchemy >= 1.4.22, != 1.4.33, < 2.0.0 # Prefect does not work with 1.4.33 and >=2.0.0 has breaking changes suds-py3==1.4.4.1 surveygizmo==1.2.3 twilio==8.2.1 diff --git a/setup.py b/setup.py index be049f9518..706b360a7f 100644 --- a/setup.py +++ b/setup.py @@ -58,10 +58,11 @@ def main(): ], "s3": ["boto3"], "salesforce": ["simple-salesforce"], + "scytl": ["defusedxml", "pytz"], "sftp": ["paramiko"], "slack": ["slackclient<2"], "smtp": ["validate-email"], - "targetsmart": ["xmltodict"], + "targetsmart": ["xmltodict", "defusedxml"], "twilio": ["twilio"], "ssh": [ "sshtunnel", From 8cafe69f5127db669878ca246b13e1f65db87274 Mon Sep 17 00:00:00 2001 From: Austin Weisgrau <62900254+austinweisgrau@users.noreply.github.com> Date: Fri, 11 Oct 2024 10:12:49 -0700 Subject: [PATCH 18/32] Allow setting gcs_tmp_bucket on GoogleBigQuery class (#1147) This parallels more closely the set up of the Redshift class. It also allows for a more simple implementation when using multiple different instances of GoogleBigQuery in the same environment. --- parsons/google/google_bigquery.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/parsons/google/google_bigquery.py b/parsons/google/google_bigquery.py index 49c17ed23a..ec1b8c5b08 100644 --- a/parsons/google/google_bigquery.py +++ b/parsons/google/google_bigquery.py @@ -143,6 +143,10 @@ class GoogleBigQuery(DatabaseConnector): A dictionary containing any requested client options. Defaults to the required scopes for making API calls against External tables stored in Google Drive. Can be set to None if these permissions are not desired + gcs_temp_bucket: str + Name of the GCS bucket that will be used for storing data during bulk transfers. + Required if you intend to perform bulk data transfers (eg. the copy_from_gcs method), + and env variable ``GCS_TEMP_BUCKET`` is not populated. """ def __init__( @@ -157,6 +161,7 @@ def __init__( "https://www.googleapis.com/auth/cloud-platform", ] }, + tmp_gcs_bucket: Optional[str] = None, ): self.app_creds = app_creds @@ -172,6 +177,7 @@ def __init__( self.project = project self.location = location self.client_options = client_options + self.tmp_gcs_bucket = tmp_gcs_bucket # We will not create the client until we need to use it, since creating the client # without valid GOOGLE_APPLICATION_CREDENTIALS raises an exception. @@ -684,7 +690,8 @@ def copy_s3( The GoogleCloudStorage Connector to use for loading data into Google Cloud Storage. tmp_gcs_bucket: str The name of the Google Cloud Storage bucket to use to stage the data to load - into BigQuery. Required if `GCS_TEMP_BUCKET` is not specified. + into BigQuery. Required if `GCS_TEMP_BUCKET` is not specified or set on + the class instance. template_table: str Table name to be used as the load schema. Load operation wil use the same columns and data types as the template table. @@ -700,8 +707,12 @@ def copy_s3( """ # copy from S3 to GCS - tmp_gcs_bucket = check_env.check("GCS_TEMP_BUCKET", tmp_gcs_bucket) - gcs_client = gcs_client or GoogleCloudStorage(app_creds=self.app_creds) + tmp_gcs_bucket = ( + tmp_gcs_bucket + or self.tmp_gcs_bucket + or check_env.check("GCS_TEMP_BUCKET", tmp_gcs_bucket) + ) + gcs_client = gcs_client or GoogleCloudStorage() temp_blob_uri = gcs_client.copy_s3_to_gcs( aws_source_bucket=bucket, aws_access_key_id=aws_access_key_id, @@ -767,7 +778,8 @@ def copy( the job fails. tmp_gcs_bucket: str The name of the Google Cloud Storage bucket to use to stage the data to load - into BigQuery. Required if `GCS_TEMP_BUCKET` is not specified. + into BigQuery. Required if `GCS_TEMP_BUCKET` is not specified or set on + the class instance. gcs_client: object The GoogleCloudStorage Connector to use for loading data into Google Cloud Storage. job_config: object @@ -783,7 +795,11 @@ def copy( client. """ data_type = "csv" - tmp_gcs_bucket = check_env.check("GCS_TEMP_BUCKET", tmp_gcs_bucket) + tmp_gcs_bucket = ( + tmp_gcs_bucket + or self.tmp_gcs_bucket + or check_env.check("GCS_TEMP_BUCKET", tmp_gcs_bucket) + ) if not tmp_gcs_bucket: raise ValueError( "Must set GCS_TEMP_BUCKET environment variable or pass in tmp_gcs_bucket parameter" From c4c9a75eb9efb5b422d784a53147db7f925345d0 Mon Sep 17 00:00:00 2001 From: Austin Weisgrau <62900254+austinweisgrau@users.noreply.github.com> Date: Fri, 11 Oct 2024 10:13:42 -0700 Subject: [PATCH 19/32] Allow passing google auth Credentials directly to GCS connector (#1148) Among other things, this enables authenticating the GCS connector using gcloud oauth from the parent shell, rather than requiring the use of service account credentials. --- parsons/google/google_cloud_storage.py | 46 ++++++++++++++++++++------ 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/parsons/google/google_cloud_storage.py b/parsons/google/google_cloud_storage.py index e92d47572e..d991bf09fe 100644 --- a/parsons/google/google_cloud_storage.py +++ b/parsons/google/google_cloud_storage.py @@ -1,14 +1,15 @@ import datetime import gzip -import petl import logging import time import uuid import zipfile -from typing import Optional +from typing import Optional, Union import google +import petl from google.cloud import storage, storage_transfer +from google.oauth2.credentials import Credentials from parsons.google.utilities import ( load_google_application_credentials, @@ -20,9 +21,11 @@ class GoogleCloudStorage(object): - """ - This class requires application credentials in the form of a json. It can be passed - in the following ways: + """Google Cloud Storage connector utility + + This class requires application credentials in the form of a + json or google oauth2 Credentials object. It can be passed in the + following ways: * Set an environmental variable named ``GOOGLE_APPLICATION_CREDENTIALS`` with the local path to the credentials json. @@ -33,22 +36,43 @@ class GoogleCloudStorage(object): * Pass in a json string using the ``app_creds`` argument. + * Generate the google credentials object directly, pass in using the + ``app_creds`` argument. + + For example, to pass in credentials from a parent shell that is + authenticated with gcloud auth: + ``` + from google.auth import default + + app_creds, _ = default() + + gcs = GoogleCloudStorage(app_creds=app_creds) + ``` + `Args:` - app_creds: str + app_creds: str, dict, or google.oauth2.credentials.Credentials object A credentials json string or a path to a json file. Not required - if ``GOOGLE_APPLICATION_CREDENTIALS`` env variable set. + if ``GOOGLE_APPLICATION_CREDENTIALS`` env variable set. Can also + pass a google oauth2 Credentials object directly. project: str The project which the client is acting on behalf of. If not passed then will use the default inferred environment. `Returns:` GoogleCloudStorage Class + """ - def __init__(self, app_creds=None, project=None): - env_credentials_path = str(uuid.uuid4()) - setup_google_application_credentials(app_creds, target_env_var_name=env_credentials_path) - credentials = load_google_application_credentials(env_credentials_path) + def __init__(self, app_creds: Optional[Union[str, dict, Credentials]] = None, project=None): + if isinstance(app_creds, Credentials): + credentials = app_creds + else: + env_credentials_path = str(uuid.uuid4()) + setup_google_application_credentials( + app_creds, target_env_var_name=env_credentials_path + ) + credentials = load_google_application_credentials(env_credentials_path) + self.project = project # Throws an error if you pass project=None, so adding if/else statement. From 941c6a4a9d7ab979bb0242caf910dfaad4c15c7b Mon Sep 17 00:00:00 2001 From: ydamit <29988641+ydamit@users.noreply.github.com> Date: Sat, 12 Oct 2024 17:27:54 -0400 Subject: [PATCH 20/32] Remove entity from campaign method (#1140) * introduce method for removing an entity record from a campaign * lint with black * lint with ruff --------- Co-authored-by: Shauna Gordon-McKeon --- parsons/action_builder/action_builder.py | 20 +++++++++++++++++++ .../test_action_builder.py | 14 +++++++++++++ 2 files changed, 34 insertions(+) diff --git a/parsons/action_builder/action_builder.py b/parsons/action_builder/action_builder.py index 5249213a7b..12d0f6f69f 100644 --- a/parsons/action_builder/action_builder.py +++ b/parsons/action_builder/action_builder.py @@ -246,6 +246,26 @@ def update_entity_record(self, identifier, data, campaign=None): return self._upsert_entity(data=data, campaign=campaign) + def remove_entity_record_from_campaign(self, identifier, campaign=None): + """ + Remove an entity record from a campaign. Records cannot be permanently deleted, but a + record that has been removed from a campaign will not appear in the UI. + `Args:` + identifier: str + The unique identifier for the record being removed. ID strings will need to begin + with the origin system, followed by a colon, e.g. `action_builder:abc123-...`. + campaign: str + Optional. The 36-character "interact ID" of the campaign whose data is to be + retrieved or edited. Not necessary if supplied when instantiating the class. + `Returns:` + Dict with HTTP response. + """ + + campaign = self._campaign_check(campaign) + + url = f"campaigns/{campaign}/people/{identifier}" + return self.api.delete_request(url=url) + def add_section_field_values_to_record(self, identifier, section, field_values, campaign=None): """ Add one or more tags (i.e. custom field value) to an existing entity record in Action diff --git a/test/test_action_builder/test_action_builder.py b/test/test_action_builder/test_action_builder.py index d0a71dd2e5..da2fb2a102 100644 --- a/test/test_action_builder/test_action_builder.py +++ b/test/test_action_builder/test_action_builder.py @@ -343,6 +343,20 @@ def test_update_entity_record(self, m): self.assertEqual(person_comp, update_response_comp) + @requests_mock.Mocker() + def test_remove_entity_record_from_campaign(self, m): + m.delete( + f"{self.api_url}/people/{self.fake_entity_id}", + json="{'message': 'Entity has been removed from the campaign'}", + ) + + remove_response = self.bldr.remove_entity_record_from_campaign(self.fake_entity_id) + + self.assertEqual( + remove_response, + "{'message': 'Entity has been removed from the campaign'}", + ) + def tagging_callback(self, request, context): # Internal method for returning the constructed tag data to test From 99decfa201c77e07090c700a9f92d41d33cac2b4 Mon Sep 17 00:00:00 2001 From: Austin Weisgrau <62900254+austinweisgrau@users.noreply.github.com> Date: Tue, 15 Oct 2024 13:30:50 -0700 Subject: [PATCH 21/32] Postgres copy explicates columns and column order (#1137) * Postgres copy explicates columns and column order * Put double quotes around table name in copy statement Necessary in certain situations, e.g. table name has capital letters * Don't lower-case table name to check if table exists Some tables have uppercase letters in their name * Overwrite some inherited methods on PostgresTable Some postgres columns (e.g. if they have a capital letter) need to be surrounded by double quotes * get_updated_rows pulls all rows if cutoff_value is None without this change, the query fails to parse --- parsons/databases/postgres/postgres.py | 41 ++++++++++++++++++++- parsons/databases/postgres/postgres_core.py | 4 +- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/parsons/databases/postgres/postgres.py b/parsons/databases/postgres/postgres.py index fc1c8c1642..18c2a61135 100644 --- a/parsons/databases/postgres/postgres.py +++ b/parsons/databases/postgres/postgres.py @@ -3,6 +3,7 @@ from parsons.databases.alchemy import Alchemy from parsons.databases.database_connector import DatabaseConnector from parsons.etl.table import Table +from typing import Optional import logging import os @@ -87,7 +88,7 @@ def copy( self.query_with_connection(sql, connection, commit=False) logger.info(f"{table_name} created.") - sql = f"COPY {table_name} FROM STDIN CSV HEADER;" + sql = f"""COPY "{table_name}" ("{'","'.join(tbl.columns)}") FROM STDIN CSV HEADER;""" with self.cursor(connection) as cursor: cursor.copy_expert(sql, open(tbl.to_csv(), "r")) @@ -102,4 +103,40 @@ def table(self, table_name): class PostgresTable(BaseTable): # Postgres table object. - pass + def max_value(self, column: str): + """Get the max value of this column from the table.""" + return self.db.query( + f""" + SELECT "{column}" + FROM {self.table} + ORDER BY "{column}" DESC + LIMIT 1 + """ + ).first + + def get_updated_rows( + self, + updated_at_column: str, + cutoff_value, + offset: int = 0, + chunk_size: Optional[int] = None, + ) -> Table: + """Get rows that have a greater updated_at_column value than the one provided.""" + sql = f""" + SELECT * + FROM {self.table} + """ + parameters = [] + + if cutoff_value is not None: + sql += f'WHERE "{updated_at_column}" > %s' + parameters.append(cutoff_value) + + if chunk_size: + sql += f" LIMIT {chunk_size}" + + sql += f" OFFSET {offset}" + + result = self.db.query(sql, parameters=parameters) + + return result diff --git a/parsons/databases/postgres/postgres_core.py b/parsons/databases/postgres/postgres_core.py index 080b35a4a9..b19c69890f 100644 --- a/parsons/databases/postgres/postgres_core.py +++ b/parsons/databases/postgres/postgres_core.py @@ -226,9 +226,9 @@ def table_exists_with_connection(self, table_name, connection, view=True): # Extract the table and schema from this. If no schema is detected then # will default to the public schema. try: - schema, table = table_name.lower().split(".", 1) + schema, table = table_name.split(".", 1) except ValueError: - schema, table = "public", table_name.lower() + schema, table = "public", table_name with self.cursor(connection) as cursor: # Check in pg tables for the table From bfb63e7c3b92595de8353012384180d225efbb6e Mon Sep 17 00:00:00 2001 From: Austin Weisgrau <62900254+austinweisgrau@users.noreply.github.com> Date: Tue, 15 Oct 2024 13:32:17 -0700 Subject: [PATCH 22/32] Fix NGPVAN _people_search arbitrary fields feature (#1150) kwargs were being passed to the wrong variable --- parsons/ngpvan/people.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parsons/ngpvan/people.py b/parsons/ngpvan/people.py index 006e2d555f..2af92df60d 100644 --- a/parsons/ngpvan/people.py +++ b/parsons/ngpvan/people.py @@ -343,7 +343,7 @@ def _people_search( id = match_json["vanId"] if kwargs: - match_json.update(kwargs) + json.update(kwargs) url = "people/" From 3f14ac5d659a0c7987012dcb843fd76dde3d3916 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Oct 2024 16:36:31 -0400 Subject: [PATCH 23/32] Bump github/codeql-action from 3.26.6 to 3.26.12 (#1151) Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3.26.6 to 3.26.12. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/4dd16135b69a43b6c8efb853346f8437d92d3c93...c36620d31ac7c881962c3d9dd939c40ec9434f2b) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/security_scorecard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/security_scorecard.yml b/.github/workflows/security_scorecard.yml index 3519c61bd0..421cb75a4e 100644 --- a/.github/workflows/security_scorecard.yml +++ b/.github/workflows/security_scorecard.yml @@ -74,6 +74,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard (optional). # Commenting out will disable upload of results to your repo's Code Scanning dashboard - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@4dd16135b69a43b6c8efb853346f8437d92d3c93 # v3.26.6 + uses: github/codeql-action/upload-sarif@c36620d31ac7c881962c3d9dd939c40ec9434f2b # v3.26.12 with: sarif_file: results.sarif \ No newline at end of file From a335313a190bfe3155bcedf0efbd7323e3c312a2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Oct 2024 16:37:04 -0400 Subject: [PATCH 24/32] Bump actions/checkout from 4.1.1 to 4.2.1 (#1152) Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.1 to 4.2.1. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.1.1...eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/dependency-review.yml | 2 +- .github/workflows/python-checks.yml | 12 ++++++------ .github/workflows/security_scorecard.yml | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 3cfec7b486..603bd4751a 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -23,6 +23,6 @@ jobs: egress-policy: audit - name: 'Checkout Repository' - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 - name: 'Dependency Review' uses: actions/dependency-review-action@5a2ce3f5b92ee19cbb1541a4984c76d921601d7c # v4.3.4 \ No newline at end of file diff --git a/.github/workflows/python-checks.yml b/.github/workflows/python-checks.yml index dfbb5c3a14..53658de6fd 100644 --- a/.github/workflows/python-checks.yml +++ b/.github/workflows/python-checks.yml @@ -22,7 +22,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 @@ -47,7 +47,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 - name: Set up Python 3.12 uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 @@ -69,7 +69,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 - name: Set up Python 3.12 uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 @@ -91,7 +91,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 - name: Set up Python 3.12 uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 @@ -113,7 +113,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 - name: Set up Python 3.12 uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 @@ -152,7 +152,7 @@ jobs: with: egress-policy: audit - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 diff --git a/.github/workflows/security_scorecard.yml b/.github/workflows/security_scorecard.yml index 421cb75a4e..dfad337384 100644 --- a/.github/workflows/security_scorecard.yml +++ b/.github/workflows/security_scorecard.yml @@ -38,7 +38,7 @@ jobs: egress-policy: audit - name: "Checkout code" - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 with: persist-credentials: false From f05d083f2e8e9e5444781995ce173d26659557e2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Oct 2024 16:37:45 -0400 Subject: [PATCH 25/32] Bump google-cloud-bigquery from 3.23.1 to 3.26.0 in /docs (#1153) Bumps [google-cloud-bigquery](https://github.com/googleapis/python-bigquery) from 3.23.1 to 3.26.0. - [Release notes](https://github.com/googleapis/python-bigquery/releases) - [Changelog](https://github.com/googleapis/python-bigquery/blob/main/CHANGELOG.md) - [Commits](https://github.com/googleapis/python-bigquery/compare/v3.23.1...v3.26.0) --- updated-dependencies: - dependency-name: google-cloud-bigquery dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5b749fb8f6..f34d598af6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ facebook-business==13.0.0 google-api-core==2.19.2 google-api-python-client==1.7.7 google-auth==2.29.0 -google-cloud-bigquery==3.23.1 +google-cloud-bigquery==3.26.0 google-cloud-storage-transfer==1.9.1 google-cloud-storage==2.16.0 google-resumable-media==2.7.0 From e89a232f6c21823ad36990ed31bcc0f79e9fb949 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Oct 2024 16:38:13 -0400 Subject: [PATCH 26/32] Bump xmltodict from 0.11.0 to 0.14.1 in /docs (#1155) Bumps [xmltodict](https://github.com/martinblech/xmltodict) from 0.11.0 to 0.14.1. - [Changelog](https://github.com/martinblech/xmltodict/blob/master/CHANGELOG.md) - [Commits](https://github.com/martinblech/xmltodict/compare/v0.11.0...v0.14.1) --- updated-dependencies: - dependency-name: xmltodict dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f34d598af6..7fc2ca1e83 100644 --- a/requirements.txt +++ b/requirements.txt @@ -42,7 +42,7 @@ surveygizmo==1.2.3 twilio==8.2.1 urllib3==1.26.19 validate-email==1.3 -xmltodict==0.11.0 +xmltodict==0.14.1 # Stuff for TMC scripts # TODO Remove when we have a TMC-specific Docker image From f62a3760b2ea7be8036791ee74d0104e8ae793e8 Mon Sep 17 00:00:00 2001 From: Paul Anzel Date: Tue, 15 Oct 2024 15:49:12 -0500 Subject: [PATCH 27/32] Add email and contact notes to NGPVAN docs (#1131) Co-authored-by: Shauna Gordon-McKeon --- docs/ngpvan.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/ngpvan.rst b/docs/ngpvan.rst index a336a5ef71..0068c4c7d1 100755 --- a/docs/ngpvan.rst +++ b/docs/ngpvan.rst @@ -357,12 +357,24 @@ Codes .. autoclass:: parsons.ngpvan.van.Codes :inherited-members: +===== +Contact Notes +===== +.. autoclass:: parsons.ngpvan.van.ContactNotes + :inherited-members: + ============= Custom Fields ============= .. autoclass:: parsons.ngpvan.van.CustomFields :inherited-members: +====== +Email +====== +.. autoclass:: parsons.ngpvan.van.Email + :inherited-members: + ====== Events ====== From 17c0dfc1b760331809ef41cf61960221abd96aaa Mon Sep 17 00:00:00 2001 From: Austin Weisgrau <62900254+austinweisgrau@users.noreply.github.com> Date: Tue, 15 Oct 2024 18:21:02 -0700 Subject: [PATCH 28/32] SFTP client for CatalistMatch has 10 minute timeout (#1149) Co-authored-by: Shauna Gordon-McKeon --- parsons/catalist/catalist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parsons/catalist/catalist.py b/parsons/catalist/catalist.py index a67f017cdb..99f2fce776 100644 --- a/parsons/catalist/catalist.py +++ b/parsons/catalist/catalist.py @@ -73,7 +73,7 @@ def __init__( token_url="https://auth.catalist.us/oauth/token", auto_refresh_url="https://auth.catalist.us/oauth/token", ) - self.sftp = SFTP("t.catalist.us", sftp_username, sftp_password) + self.sftp = SFTP("t.catalist.us", sftp_username, sftp_password, timeout=7200) def load_table_to_sftp(self, table: Table, input_subfolder: Optional[str] = None) -> str: """Load table to Catalist sftp bucket as gzipped CSV for matching. From c148909e8191a4f20a419fc9a4e4400121bcdb6d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 16 Oct 2024 11:12:00 -0400 Subject: [PATCH 29/32] Bump install-pinned/uv (#1154) Bumps [install-pinned/uv](https://github.com/install-pinned/uv) from de03c60d508703a83d3f8f49afcf1249590ecda1 to 03a68782c27167b267490a9392ac24d6b93a2f14. - [Release notes](https://github.com/install-pinned/uv/releases) - [Commits](https://github.com/install-pinned/uv/compare/de03c60d508703a83d3f8f49afcf1249590ecda1...03a68782c27167b267490a9392ac24d6b93a2f14) --- updated-dependencies: - dependency-name: install-pinned/uv dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/python-checks.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-checks.yml b/.github/workflows/python-checks.yml index 53658de6fd..52ec7dfd4c 100644 --- a/.github/workflows/python-checks.yml +++ b/.github/workflows/python-checks.yml @@ -30,7 +30,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install uv - uses: install-pinned/uv@de03c60d508703a83d3f8f49afcf1249590ecda1 # 0.4.12 + uses: install-pinned/uv@03a68782c27167b267490a9392ac24d6b93a2f14 # 0.4.12 - name: Install dependencies env: @@ -55,7 +55,7 @@ jobs: python-version: "3.12" - name: Install uv - uses: install-pinned/uv@de03c60d508703a83d3f8f49afcf1249590ecda1 # 0.4.12 + uses: install-pinned/uv@03a68782c27167b267490a9392ac24d6b93a2f14 # 0.4.12 - name: Install dependencies run: | @@ -77,7 +77,7 @@ jobs: python-version: "3.12" - name: Install uv - uses: install-pinned/uv@de03c60d508703a83d3f8f49afcf1249590ecda1 # 0.4.12 + uses: install-pinned/uv@03a68782c27167b267490a9392ac24d6b93a2f14 # 0.4.12 - name: Install dependencies run: | @@ -99,7 +99,7 @@ jobs: python-version: "3.12" - name: Install uv - uses: install-pinned/uv@de03c60d508703a83d3f8f49afcf1249590ecda1 # 0.4.12 + uses: install-pinned/uv@03a68782c27167b267490a9392ac24d6b93a2f14 # 0.4.12 - name: Install bandit run: | @@ -121,7 +121,7 @@ jobs: python-version: "3.12" - name: Install uv - uses: install-pinned/uv@de03c60d508703a83d3f8f49afcf1249590ecda1 # 0.4.12 + uses: install-pinned/uv@03a68782c27167b267490a9392ac24d6b93a2f14 # 0.4.12 - name: Install dependencies run: | From cc1aa2ad4571cb1d5a0de7a335af81713e40a0a5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 16 Oct 2024 11:12:19 -0400 Subject: [PATCH 30/32] Bump step-security/harden-runner from 2.9.1 to 2.10.1 (#1156) Bumps [step-security/harden-runner](https://github.com/step-security/harden-runner) from 2.9.1 to 2.10.1. - [Release notes](https://github.com/step-security/harden-runner/releases) - [Commits](https://github.com/step-security/harden-runner/compare/5c7944e73c4c2a096b17a9cb74d65b6c2bbafbde...91182cccc01eb5e619899d80e4e971d6181294a7) --- updated-dependencies: - dependency-name: step-security/harden-runner dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/dependency-review.yml | 2 +- .github/workflows/python-checks.yml | 2 +- .github/workflows/security_scorecard.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 603bd4751a..9f0c1f93e4 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@5c7944e73c4c2a096b17a9cb74d65b6c2bbafbde # v2.9.1 + uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 with: egress-policy: audit diff --git a/.github/workflows/python-checks.yml b/.github/workflows/python-checks.yml index 52ec7dfd4c..433eb99c6e 100644 --- a/.github/workflows/python-checks.yml +++ b/.github/workflows/python-checks.yml @@ -148,7 +148,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@5c7944e73c4c2a096b17a9cb74d65b6c2bbafbde # v2.9.1 + uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 with: egress-policy: audit diff --git a/.github/workflows/security_scorecard.yml b/.github/workflows/security_scorecard.yml index dfad337384..85eac5f48f 100644 --- a/.github/workflows/security_scorecard.yml +++ b/.github/workflows/security_scorecard.yml @@ -33,7 +33,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@5c7944e73c4c2a096b17a9cb74d65b6c2bbafbde # v2.9.1 + uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 with: egress-policy: audit From a4cf070257029494676fc08f55ec496955ed36b3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 16 Oct 2024 11:12:39 -0400 Subject: [PATCH 31/32] Bump pygithub from 1.51 to 2.4.0 (#1157) Bumps [pygithub](https://github.com/pygithub/pygithub) from 1.51 to 2.4.0. - [Release notes](https://github.com/pygithub/pygithub/releases) - [Changelog](https://github.com/PyGithub/PyGithub/blob/main/doc/changes.rst) - [Commits](https://github.com/pygithub/pygithub/compare/v1.51...v2.4.0) --- updated-dependencies: - dependency-name: pygithub dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7fc2ca1e83..a8af914744 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,7 +28,7 @@ oauth2client==4.1.3 paramiko==3.4.0 petl==1.7.15 psycopg2-binary==2.9.9 -PyGitHub==1.51 +PyGitHub==2.4.0 python-dateutil==2.8.2 requests==2.32.3 requests_oauthlib==1.3.0 From a9b45c250ab88188ca98b13e336a6a766ff9d00a Mon Sep 17 00:00:00 2001 From: Shauna Gordon-McKeon Date: Wed, 16 Oct 2024 12:24:11 -0400 Subject: [PATCH 32/32] Revert "Bump install-pinned/uv (#1154)" (#1168) This reverts commit c148909e8191a4f20a419fc9a4e4400121bcdb6d. --- .github/workflows/python-checks.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-checks.yml b/.github/workflows/python-checks.yml index 433eb99c6e..a0835217e0 100644 --- a/.github/workflows/python-checks.yml +++ b/.github/workflows/python-checks.yml @@ -30,7 +30,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install uv - uses: install-pinned/uv@03a68782c27167b267490a9392ac24d6b93a2f14 # 0.4.12 + uses: install-pinned/uv@de03c60d508703a83d3f8f49afcf1249590ecda1 # 0.4.12 - name: Install dependencies env: @@ -55,7 +55,7 @@ jobs: python-version: "3.12" - name: Install uv - uses: install-pinned/uv@03a68782c27167b267490a9392ac24d6b93a2f14 # 0.4.12 + uses: install-pinned/uv@de03c60d508703a83d3f8f49afcf1249590ecda1 # 0.4.12 - name: Install dependencies run: | @@ -77,7 +77,7 @@ jobs: python-version: "3.12" - name: Install uv - uses: install-pinned/uv@03a68782c27167b267490a9392ac24d6b93a2f14 # 0.4.12 + uses: install-pinned/uv@de03c60d508703a83d3f8f49afcf1249590ecda1 # 0.4.12 - name: Install dependencies run: | @@ -99,7 +99,7 @@ jobs: python-version: "3.12" - name: Install uv - uses: install-pinned/uv@03a68782c27167b267490a9392ac24d6b93a2f14 # 0.4.12 + uses: install-pinned/uv@de03c60d508703a83d3f8f49afcf1249590ecda1 # 0.4.12 - name: Install bandit run: | @@ -121,7 +121,7 @@ jobs: python-version: "3.12" - name: Install uv - uses: install-pinned/uv@03a68782c27167b267490a9392ac24d6b93a2f14 # 0.4.12 + uses: install-pinned/uv@de03c60d508703a83d3f8f49afcf1249590ecda1 # 0.4.12 - name: Install dependencies run: |