Python runtime and package updates

An attempt to address intermittent uploader errors - bump runtime from 3.8 to 3.10 - update to latest requests & urllib3, plus several downstream deps - pinned the cdk library until we have some pressing need to update - also a couple of packages flagged by pip-audit - added a python version matrix to the github workflow (although it's currently only testing 3.10)
harvard-dce · Dec 7, 2024 · 00b894f · 00b894f
1 parent 85e5d21
commit 00b894f
Show file tree

Hide file tree

Showing 13 changed files with 441 additions and 401 deletions.
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
@@ -10,33 +10,45 @@ env:
 
 jobs:
   unit-tests:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: true
+      matrix:
+        python-version: ['3.10']
+
     steps:
       - name: Checkout Repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.8.16
+          python-version: ${{ matrix.python-version }}
           cache: 'pip'
           cache-dependency-path: './requirements/dev.txt'
 
       - name: Install libraries
         run: pip install -r requirements/dev.txt
 
       - name: Run tests
-        run: tox
+        run: tox -e py
 
   lint-check:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: true
+      matrix:
+        python-version: ['3.10']
+
     steps:
       - name: Checkout Repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
-          python-version: 3.8.16
+          python-version: ${{ matrix.python-version }}
           cache: 'pip'
           cache-dependency-path: './requirements/dev.txt'
 

diff --git a/.gitignore b/.gitignore
@@ -13,3 +13,4 @@ outfile.txt
 .cdk.staging
 cdk.out
 cdk.context.json
+node_modules
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [unreleased]
 
+### Changed
+
+- bumped python runtime from 3.8 to 3.10
+- updated package versions for requests, urllib3, certifi and idna
+
 ## [4.2.4 - 2024-08-02]
 
 ### Changed

diff --git a/cdk/function.py b/cdk/function.py
@@ -36,7 +36,7 @@ def __init__(
 
         function_props = {
             "function_name": f"{self.stack_name}-{name}",
-            "runtime": aws_lambda.Runtime.PYTHON_3_8,
+            "runtime": aws_lambda.Runtime.PYTHON_3_10,
             "code": aws_lambda.Code.from_bucket(
                 bucket=lambda_code_bucket, key=f"{self.stack_name}/{name}.zip"
             ),

diff --git a/function_requirements/common-requirements.in b/function_requirements/common-requirements.in
@@ -1,6 +1,7 @@
 boto3
 aws-lambda-logging
 requests
+urllib3>2
 tenacity
 PyJWT
 pytz

diff --git a/function_requirements/common-requirements.txt b/function_requirements/common-requirements.txt
@@ -6,91 +6,91 @@
 #
 aws-lambda-logging==0.1.1
     # via -r function_requirements/common-requirements.in
-boto3==1.26.142
+boto3==1.35.76
     # via -r function_requirements/common-requirements.in
-botocore==1.29.142
+botocore==1.35.76
     # via
     #   boto3
     #   s3transfer
-cachetools==5.3.0
+cachetools==5.5.0
     # via google-auth
-certifi==2023.5.7
+certifi==2024.8.30
     # via requests
-charset-normalizer==3.1.0
+charset-normalizer==3.4.0
     # via requests
-google-api-core==2.11.0
+google-api-core==2.23.0
     # via google-api-python-client
-google-api-python-client==2.87.0
+google-api-python-client==2.154.0
     # via -r function_requirements/common-requirements.in
-google-auth==2.19.0
+google-auth==2.36.0
     # via
     #   -r function_requirements/common-requirements.in
     #   google-api-core
     #   google-api-python-client
     #   google-auth-httplib2
     #   google-auth-oauthlib
-google-auth-httplib2==0.1.0
+google-auth-httplib2==0.2.0
     # via google-api-python-client
-google-auth-oauthlib==1.0.0
+google-auth-oauthlib==1.2.1
     # via -r function_requirements/common-requirements.in
-googleapis-common-protos==1.59.0
+googleapis-common-protos==1.66.0
     # via google-api-core
 httplib2==0.22.0
     # via
     #   google-api-python-client
     #   google-auth-httplib2
-idna==3.4
+idna==3.10
     # via requests
 jmespath==1.0.1
     # via
     #   boto3
     #   botocore
 oauthlib==3.2.2
     # via requests-oauthlib
-protobuf==4.23.2
+proto-plus==1.25.0
+    # via google-api-core
+protobuf==5.29.1
     # via
     #   google-api-core
     #   googleapis-common-protos
-pyasn1==0.5.0
+    #   proto-plus
+pyasn1==0.6.1
     # via
     #   pyasn1-modules
     #   rsa
-pyasn1-modules==0.3.0
+pyasn1-modules==0.4.1
     # via google-auth
-pyjwt==2.7.0
+pyjwt==2.10.1
     # via -r function_requirements/common-requirements.in
-pyparsing==3.0.9
+pyparsing==3.2.0
     # via httplib2
-python-dateutil==2.8.2
+python-dateutil==2.9.0.post0
     # via botocore
-python-dotenv==1.0.0
+python-dotenv==1.0.1
     # via -r function_requirements/common-requirements.in
-pytz==2023.3
+pytz==2024.2
     # via -r function_requirements/common-requirements.in
-requests==2.31.0
+requests==2.32.3
     # via
     #   -r function_requirements/common-requirements.in
     #   google-api-core
     #   requests-oauthlib
-requests-oauthlib==1.3.1
+requests-oauthlib==2.0.0
     # via google-auth-oauthlib
 rsa==4.7.2
     # via
     #   -r function_requirements/common-requirements.in
     #   google-auth
-s3transfer==0.6.1
+s3transfer==0.10.4
     # via boto3
-six==1.16.0
-    # via
-    #   google-auth
-    #   google-auth-httplib2
-    #   python-dateutil
-tenacity==8.5.0
+six==1.17.0
+    # via python-dateutil
+tenacity==9.0.0
     # via -r function_requirements/common-requirements.in
 uritemplate==4.1.1
     # via google-api-python-client
-urllib3==1.26.16
+urllib3==2.2.3
     # via
+    #   -r function_requirements/common-requirements.in
     #   botocore
-    #   google-auth
     #   requests
diff --git a/functions/utils/status.py b/functions/utils/status.py
@@ -154,9 +154,9 @@ def set_pipeline_status(
                 "OR pipeline_state = :recording_stopped "
             )
         elif state in ZoomStatus:
-            expression_attribute_values[
-                ":recording_processing"
-            ] = ZoomStatus.RECORDING_PROCESSING.name
+            expression_attribute_values[":recording_processing"] = (
+                ZoomStatus.RECORDING_PROCESSING.name
+            )
 
             # Enforce cannot transition back to a recording in progress state
             # from recording processing

diff --git a/functions/zoom-uploader.py b/functions/zoom-uploader.py
@@ -69,7 +69,8 @@ def oc_api_request(method, endpoint, **kwargs):
     logger.info({"url": url, "kwargs": kwargs})
     try:
         resp = session.request(method, url, **kwargs)
-    except requests.RequestException:
+    except requests.RequestException as e:
+        logger.exception(f"requests raised an exception: {e}")
         raise OpencastConnectionError
     resp.raise_for_status()
     return resp

diff --git a/requirements/base.in b/requirements/base.in
@@ -5,7 +5,7 @@ boto3
 botocore
 requests
 tabulate
-aws-cdk-lib
+aws-cdk-lib==2.81.0
 constructs
 tenacity
 pyyaml!=6.0.0,!=5.4.0,!=5.4.1, # pyyaml later versions are broken with cython 3