From a57cbc44a3e24bcde5b970f07c2112fc6df3d021 Mon Sep 17 00:00:00 2001 From: Andreas Abel Date: Thu, 28 Oct 2021 10:19:29 +0200 Subject: [PATCH] Re #7777 #7778: check requirements.txt for security advisory There is a new make file, doc/Makefile that defines these goals: - `check-requirements`: Check `requirements.txt` for security problems (CVEs) using `skjold`. This goal is intended for the "Users guide" CI. SKJOLD_GITHUB_API_TOKEN might have to be set if GITHUB_TOKEN is not in the environment, in order to access the GitHub GraphQL API. - `build-and-check-requirements`: Rebuild `requirements.txt` from `requirements.in` using `pip-compile`, and check with `check-requirements`. This goal is intended for manual invocation. It is invoked from the top Makefile via goal `users-guide-requirements`. Alternatively, these goals could be coupled with the doc build `make users-guide`. However, since these goals require a couple of seconds to run, I think it is annoying to call them on every build of the documentation. --- .github/workflows/users-guide.yml | 14 ++++++++++ .gitignore | 6 ++--- Makefile | 9 +++++++ doc/Makefile | 43 +++++++++++++++++++++++++++++++ doc/pyproject.toml | 11 ++++++++ doc/requirements.in | 2 ++ doc/requirements.txt | 6 +++-- 7 files changed, 86 insertions(+), 5 deletions(-) create mode 100644 doc/Makefile create mode 100644 doc/pyproject.toml diff --git a/.github/workflows/users-guide.yml b/.github/workflows/users-guide.yml index 3f75d5d24d7..9bed41273f4 100644 --- a/.github/workflows/users-guide.yml +++ b/.github/workflows/users-guide.yml @@ -7,6 +7,9 @@ on: branches: - master paths: + - 'doc/Makefile' + - 'doc/pyproject.toml' + - 'doc/requirements.in' - 'doc/requirements.txt' - 'doc/*.inc' - 'doc/*.py' @@ -15,6 +18,9 @@ on: - '.github/workflows/users-guide.yml' pull_request: paths: + - 'doc/Makefile' + - 'doc/pyproject.toml' + - 'doc/requirements.in' - 'doc/requirements.txt' - 'doc/*.inc' - 'doc/*.py' @@ -25,6 +31,10 @@ on: types: - created +defaults: + run: + shell: bash + jobs: build: if: | @@ -59,3 +69,7 @@ jobs: with: name: users-guide-html path: html/ + + - name: Check security of requirements.txt + run: | + make SKJOLD_GITHUB_API_TOKEN=${{ secrets.GITHUB_TOKEN }} -C doc check-requirements diff --git a/.gitignore b/.gitignore index 4747b90b7af..efb4f1dd856 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -# trivial gitignore file .cabal-sandbox/ cabal.sandbox.config cabal.project.local @@ -15,7 +14,6 @@ dist-* register.sh ./cabal.config cabal-tests.log -.python-sphinx-virtualenv/ /Cabal/dist/ /Cabal/tests/Setup @@ -68,6 +66,8 @@ register.sh # python artifacts from documentation builds *.pyc +.python-sphinx-virtualenv/ +/doc/.skjold_cache/ # macOS folder metadata .DS_Store @@ -76,4 +76,4 @@ register.sh bench.html # Emacs -.projectile \ No newline at end of file +.projectile diff --git a/Makefile b/Makefile index 99eeb178726..3a78bb4b2d3 100644 --- a/Makefile +++ b/Makefile @@ -251,3 +251,12 @@ $(USERGUIDE_STAMP) : doc/*.rst .python-sphinx-virtualenv: python3 -m venv .python-sphinx-virtualenv (. ./.python-sphinx-virtualenv/bin/activate) + +# This goal is intended for manual invocation, always rebuilds. +.PHONY: users-guide-requirements +users-guide-requirements: doc/requirements.txt + +.PHONY: doc/requirements.txt +doc/requirements.txt: .python-sphinx-virtualenv + . .python-sphinx-virtualenv/bin/activate \ + && make -C doc build-and-check-requirements diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 00000000000..d0b25cce466 --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,43 @@ +# Build and safety-check requirements.txt + +# skjold needs a personal github access token. This needs no permissions, +# it is only required to query the GitHub GraphQL API v4. +# See: https://pythonawesome.com/security-audit-python-project-dependencies-against-security-advisory-databases/ +# We attempt to get it from the environment variable GITHUB_TOKEN. +# It can also be passed to this Makefile via either: +# +# make GITHUB_TOKEN=... (build-and-)check-requirements +# make SKJOLD_GITHUB_API_TOKEN=... (build-and-)check-requirements +# +# +SKJOLD_GITHUB_API_TOKEN=${GITHUB_TOKEN} + +.PHONY: build-and-check-requirements +build-and-check-requirements: requirements.txt check-requirements + +# Always rebuild requirements.txt +.PHONY: requirements.txt +# requirements.txt is generated from requirements.in +# via pip-compile included in the pip-tools package. +# See https://modelpredict.com/wht-requirements-txt-is-not-enough +requirements.txt: requirements.in + . ../.python-sphinx-virtualenv/bin/activate \ + && pip install pip-tools \ + && pip-compile requirements.in + +# Check requirements.txt for security violations via skjold, +# configured in pyproject.toml. +# See: https://pythonawesome.com/security-audit-python-project-dependencies-against-security-advisory-databases/ +.PHONY: check-requirements +check-requirements: + @if [ "\'${SKJOLD_GITHUB_API_TOKEN}\'" == "\'\'" ] \ + ; then \ + echo "WARNING: Neither SKOLD_GITHUB_API_TOKEN nor GITHUB_TOKEN is set." \ + ; echo "Vulnerability check via skjold might fail when using the GitHub GraphQL API." \ + ; fi + . ../.python-sphinx-virtualenv/bin/activate \ + && pip install skjold \ + && skjold audit +# NB: For portability, we use '.' (sh etc.) instead of 'source' (bash). + +# EOF diff --git a/doc/pyproject.toml b/doc/pyproject.toml new file mode 100644 index 00000000000..a83dbeca4ba --- /dev/null +++ b/doc/pyproject.toml @@ -0,0 +1,11 @@ +# https://pythonawesome.com/security-audit-python-project-dependencies-against-security-advisory-databases/ +[tool.skjold] +sources = ['github', 'gemnasium', 'pyup'] +report_only = false + # ALT: true # Report only, always exit with zero. +report_format = 'cli' + # ALT: 'json' # Output findings as `json`. Default is 'cli'. +verbose = true +cache_dir = '.skjold_cache' +cache_expires = 43200 # Cache max. age. (43200 = 12hrs) +ignore_file = '.skjoldignore' diff --git a/doc/requirements.in b/doc/requirements.in index 3d5ffeb53fb..585618166d3 100644 --- a/doc/requirements.in +++ b/doc/requirements.in @@ -1,3 +1,5 @@ sphinx >= 3.1 sphinx_rtd_theme sphinx-jsonschema +# Pygments>=2.7.4 suggested by CVE-2021-20270 CVE-2021-27291 +Pygments >= 2.7.4 diff --git a/doc/requirements.txt b/doc/requirements.txt index 1542de013f4..f399f4c10c6 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -29,8 +29,10 @@ markupsafe==1.1.1 # via jinja2 packaging==20.9 # via sphinx -pygments==2.5.2 - # via sphinx +pygments==2.10.0 + # via + # -r requirements.in + # sphinx pyparsing==2.4.7 # via packaging pytz==2021.3