new: copy from catalog

GSA/catalog.data.gov#677

.github/workflows/snyk.yml

@@ -0,0 +1,74 @@
+---
+name: Check for Snyk Vulnerabilities
+
+on:   # yamllint disable-line rule:truthy
+  workflow_dispatch:
+  schedule:
+    - cron: '0 12 * * *'  # every day at 12pm UTC
+
+jobs:
+  snyk:
+    name: snyk test
+    runs-on: ubuntu-20.04   # Pin to have access to Python 3.8
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install Dependencies
+        run: |
+          npm install snyk -g
+          sudo apt-get update -y
+          sudo apt-get install -y \
+            openssl libssl-dev libffi-dev pkg-config libxml2-dev \
+            libxmlsec1-dev libxmlsec1-openssl libgeos-dev proj-bin
+          pip3 install -r requirements.txt
+      # yamllint disable rule:line-length
+      - name: Run Snyk Scan
+        run: |
+          # Run scan
+          snyk auth ${{ secrets.SNYK_TOKEN }}
+          snyk test --file=requirements.txt --json-file-output=scan.json || echo "Scan complete"
+
+          # Exit if no vulnerabilities
+          # Succeed, so that PR is NOT created
+          [[ "$(jq '.ok' scan.json)" == "true" ]] && exit 0
+
+          # Update requirements.in with the snyk fix suggestions
+          python bin/snyk-update.py
+
+          # Remove unnecessary file so that it doesn't get added to the repo
+          rm scan.json
+
+          # Update requirements.txt
+          make update-dependencies
+
+          # Fail so that PR is created
+          exit 1
+      - name: Create Pull Request
+        if: ${{ failure() }}
+        id: scpr
+        uses: peter-evans/create-pull-request@v4
+        with:
+          token: ${{ secrets.ADD_TO_PROJECT_PAT }}
+          commit-message: Update Pip Requirements
+          committer: Data.gov Github <[email protected]>
+          author: ${{ github.actor }} <[email protected]>
+          signoff: false
+          branch: requirement-patches
+          delete-branch: true
+          title: '[Snyk + GH Actions] Update requirements'
+          body: |
+            Update requirements
+            - Updated `requirements.in` + `requirements.txt`
+            - Auto-generated by [snyk.yml][1]
+
+            [1]: https://github.com/gsa/catalog.data.gov/.github/workflows/snyk.yml
+          labels: |
+            requirements
+            automated pr
+            snyk
+          reviewers: GSA/data-gov-team
+          team-reviewers: |
+            owners
+            maintainers
+          draft: false
+          # yamllint enable rule:line-length

bin/snyk-update.py

@@ -0,0 +1,24 @@
+
+import json
+import os
+
+# Load results
+scan_file = open('scan.json')
+scan = json.load(scan_file)
+scan_file.close()
+
+# Make changes to fixable results
+remediations = scan['remediation']['pin']
+for k, v in remediations.items():
+    package, old_version = k.split('@')
+    new_version = v['upgradeTo'].split('@')[1]
+    print(package, old_version, new_version)
+
+    # Remove old version
+    os.system('sed -i "/%s/d" requirements.in.txt' % (package + "==" + old_version))
+    os.system('sed -i "/%s/d" requirements.in.txt' % (package + ">=" + old_version))
+
+    # Add new version
+    os.system("echo '%s' >> requirements.in.txt" % (package + ">=" + new_version))
+
+# TODO: Handle unfixable results