Merge pull request #17 from edly-io/ALI/EDLY-5530

upgrade to koa release

.annotation_safe_list.yml

@@ -0,0 +1,39 @@
+# This is a Code Annotations automatically-generated Django model safelist file.
+# These models must be annotated as follows in order to be counted in the coverage report.
+# See https://code-annotations.readthedocs.io/en/latest/safelist.html for more information.
+#
+# fake_app_1.FakeModelName:
+#    ".. no_pii:": "This model has no PII"
+# fake_app_2.FakeModel2:
+#    ".. choice_annotation:": foo, bar, baz
+
+admin.LogEntry:
+  ".. no_pii:": "This model has no PII"
+auth.Group:
+  ".. no_pii:": "This model has no PII"
+auth.Permission:
+  ".. no_pii:": "This model has no PII"
+contenttypes.ContentType:
+  ".. no_pii:": "This model has no PII"
+sessions.Session:
+  ".. no_pii:": "This model has no PII"
+sites.Site:
+  ".. no_pii:": "This model has no PII"
+social_django.Association:
+  ".. no_pii:": "This model has no PII"
+social_django.Code:
+  ".. pii:": "Email address"
+  ".. pii_types:": other
+  ".. pii_retirement:": retained
+social_django.Nonce:
+  ".. no_pii:": "This model has no PII"
+social_django.Partial:
+  ".. no_pii:": "This model has no PII"
+social_django.UserSocialAuth:
+  ".. no_pii:": "This model has no PII"
+waffle.Flag:
+  ".. no_pii:": "This model has no PII"
+waffle.Sample:
+  ".. no_pii:": "This model has no PII"
+waffle.Switch:
+  ".. no_pii:": "This model has no PII"

.coveragerc

@@ -5,7 +5,6 @@ source=credentials
 omit =
     credentials/settings*
     credentials/conf*
-    credentials/apps/core/s3utils.py
     *wsgi.py
     *migrations*
     *admin.py

.dockerignore

@@ -2,7 +2,6 @@
 !.babelrc
 !.eslintrc.json
 !.isort.cfg
-!.pep8
 !.tx
 !acceptance_tests
 !credentials
@@ -19,3 +18,4 @@
 !README.rst
 !requirements.txt
 !webpack.config.js
+!setup.cfg

.gitignore

@@ -35,7 +35,6 @@ assets/
 .coverage
 htmlcov
 .tox
-nosetests.xml
 unittests.xml
 
 ### Internationalization artifacts
@@ -99,3 +98,6 @@ credentials/static/jsi18n/
 docs/_build/
 node_modules/
 webpack-stats.json
+
+# PII annotation reports
+pii_report

.pii_annotations.yml

@@ -0,0 +1,35 @@
+source_path: ./
+report_path: pii_report
+safelist_path: .annotation_safe_list.yml
+coverage_target: 100.0
+annotations:
+    ".. no_pii:":
+    "pii_group":
+        - ".. pii:":
+        - ".. pii_types:":
+            choices:
+              - id # Unique identifier for the user which is shared across systems
+              - name # Used for any part of the user’s name
+              - username
+              - password
+              - location # Used for any part of any type address or country stored
+              - phone_number # Used for phone or fax numbers
+              - email_address
+              - birth_date # Used for any part of a stored birth date
+              - ip # IP address
+              - external_service # Used for external service ids or links such as social media links or usernames, website links, etc.
+              - biography # Any type of free-form biography field
+              - gender
+              - sex
+              - image
+              - video
+              - other
+        - ".. pii_retirement:":
+            choices:
+              - retained     # Intentionally kept for legal reasons
+              - local_api    # An API exists in this repository for retiring this information
+              - consumer_api # The data's consumer must implement an API for retiring this information
+              - third_party  # A third party API exists to retire this data
+extensions:
+    python:
+        - py

.travis.yml

@@ -1,4 +1,6 @@
 language: python
+os: linux
+dist: focal
 
 branches:
   only:
@@ -9,8 +11,6 @@ env:
     - secure: sw/gGXEWETCMr3FB0mnvBRPTMJZTmL8QCzv7mYtdhXoUdFH45w0mq0KQUtKSJrNDTt2QY/pkzaF5xneT2dKnCgrkNNsPHEDnV0CUbiEmj9U/GvuRQW79ADyp/eNx+L6enM2uLAlXnd7Un8NDXcDLrYJLnt2Xnm7o9zv+481eSGIIH+f6M7ZEbHQP1TzxQnRz+2xksZa3l/d7eFK9FRLhmUnEQHlVo7r9di2TdEAQUQmuTGEj0cgAN+l9eRKL8T7gf7UN3v7DRZ7qZ5O66yKyniQ05xyzem+bZFMAxry4TM6/TZU8Ghchbne+bT77UYeZcnSKunWdhK5PUntp02HpN9kaOjh02uwXm6wTxi9/Xjt/ap65ZPCeCOZ2dPBncdnbi9OZY13eIZHweQR1hwWVJAC7n9p6/svnQ5CRruN95zc6b+ZecRfvPfMgjAd9J9lw0hkHVqC2rq4e52trJQkXH3zw9GXQ/V/NorpGCTbXDSopuFtATAvNDx5W4jUmAcjHGQioKmfSm1EPN5x78ZxPd/QBIPR+1RKGScYLaWexuw3go/LSv5PL/m2vR1mzp+OQri6sd91K+up2xIp5k/VIa+j8uWQLi68vnv4lcQYRIKDsJmM6ItobFuMR+I2UB3Zk2flCDPMCJ8I0B5YZsRaF/luzYD3hU6S5FJIpdWaexXQ=
     - secure: eRB1Jyj1uLdEV8zkZA8F6Gpw26Z9IDQj2wPKXSDrMf2b/p769Y7bssvjIKGbPS5NoPG0L0vImzTUgV5B/DPXv6O7dLvPkNumeNf1+0XLRsBbqiKarcKVia8Ppe/pUgUJytqLrBw6OyUMvNUwJVtTgl9t7j7mmZFahJMrxGpRY/VtGAT8eSxkS6dHBL2WYdE81WPYpT3wR1QMs7NObErk8U8ztOZEaF6m8FwTmk4t96O3gcgand4fj3/XFaEmVxRH4Qq2n9ByIGja6uGlVteNFL9sBcSajZavxvrlsTY1jfy59irp8y3+D0baV9LFHEvCRAPvW2fVrfV5Yyn+TtbCcEGEhuedjLQaUOKJBuakf5B0qjSMJlW6NHn/DWBS6A8j78A0p03iUaB51mrQhG3/pMaET1duIBofhjJ2XMFQYkdw6W/tC78F9qk2TX0nrqfU4tI0zhPaXYE3AhZDhiy/OCQeA/tWxSbT5G1XD6+/dhPqdG7OFqgKjPV0XrK4YVzMVQp7xHu2vQVZF/Dw3MmyE+rxw4zEB5FRCoffoqkywE714+nRoHQcaUxf+6lvvmVP8mPvDqE/Kte3RruaZrDppZtPsLPmwYPFKLvZUp8d2UFeW2wiyIP4mhkQ2lms4JiVfzDtBimx2fW3BUN7xSR14jpNUooFb9V7dADzDHL2DvI=
 
-sudo: required
-
 services:
   - docker
 
@@ -20,17 +20,8 @@ cache:
 before_install:
   - docker-compose -f docker-compose.yml -f docker-compose.travis.yml up -d
 
-matrix:
+jobs:
   include:
-    - python: 3.5
-      env: TESTNAME=quality-and-translations
-      install: true
-      script:
-        - make exec-requirements
-        - make exec-check_translations_up_to_date
-        - make exec-validate-translations
-        - make exec-quality
-        - make exec-check_keywords
     - python: 3.8
       env: TESTNAME=quality-and-translations
       install: true
@@ -40,16 +31,8 @@ matrix:
         - make exec-validate-translations
         - make exec-quality
         - make exec-check_keywords
+        - make exec-pii_check
 
-    - python: 3.5
-      env: TESTNAME=acceptance-tests
-      install:
-        - make exec-requirements
-      script:
-        - make exec-static
-        - make exec-accept
-      addons:
-          firefox: latest
     - python: 3.8
       env: TESTNAME=acceptance-tests
       install:
@@ -60,15 +43,24 @@ matrix:
       addons:
         firefox: latest
 
-    - python: 3.5
+    - python: 3.8
       env: TOXENV=django22
       install: true
       script:
         - make exec-requirements
         - make exec-static
         - make exec-tests
+
     - python: 3.8
-      env: TOXENV=django22
+      env: TOXENV=django30
+      install: true
+      script:
+        - make exec-requirements
+        - make exec-static
+        - make exec-tests
+
+    - python: 3.8
+      env: TOXENV=django31
       install: true
       script:
         - make exec-requirements

Makefile

@@ -1,8 +1,14 @@
-.DEFAULT_GOAL := tests
+.DEFAULT_GOAL := help
 NODE_BIN=./node_modules/.bin
 TOX = ''
 
-.PHONY: requirements upgrade piptools production-requirements all-requirements
+.PHONY: help clean production-requirements js-requirements all-requirements requirements isort isort_check pycodestyle \
+        quality quality-js test-react tests js-tests static static.dev static.watch migrate up up-dev up-test \
+        exec-validate-translations exec-check_translations_up_to_date exec-check_keywords exec-pii_check exec-clean \
+        exec-requirements exec-static exec-quality exec-tests exec-accept exec-validate exec-coverage html_coverage \
+        shell tail stop down accept extract_translations dummy_translations compile_translations fake_translations \
+        pull_translations push_translations detect_changed_source_translations validate_translations \
+        check_translations_up_to_date piptools upgrade check_keywords pii_check
 
 ifdef TOXENV
 TOX := tox -- #to isolate each tox environment if TOXENV is defined
@@ -11,7 +17,7 @@ endif
 # Generates a help message. Borrowed from https://github.com/pydanny/cookiecutter-djangopackage.
 help: ## Display this help message
 	@echo "Please use \`make <target>\` where <target> is one of"
-	@perl -nle'print $& if m{^[\.a-zA-Z_-]+:.*?## .*$$}' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m  %-25s\033[0m %s\n", $$1, $$2}'
+	@awk -F ':.*?## ' '/^[a-zA-Z]/ && NF==2 {printf "\033[36m  %-25s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) | sort
 
 clean: ## Remove all generated files
 	coverage erase
@@ -21,7 +27,7 @@ clean: ## Remove all generated files
 	git clean -fd credentials/conf/locale
 
 production-requirements: piptools ## Install requirements for production
-	npm install --production --no-save
+	npm install --no-save
 	pip-sync requirements.txt
 
 js-requirements: ## Install frontend requirements
@@ -36,9 +42,18 @@ requirements: piptools ## Install requirements for local development
 	npm install --unsafe-perm ## This flag exists to force node-sass to build correctly on docker. Remove as soon as possible.
 	pip-sync requirements/dev.txt
 
+isort: ## Run isort to sort imports in all Python files
+	isort --recursive --atomic acceptance_tests/ credentials/
+
+isort_check: ## Check that isort has been run
+	isort --check-only -rc acceptance_tests/ credentials/
+
+pycodestyle: ## Run pycodestyle
+	pycodestyle acceptance_tests credentials *.py
+
 quality: ## Run linters
-	isort --check-only --recursive acceptance_tests/ credentials/
-	pep8 --config=.pep8 acceptance_tests credentials *.py
+	make isort_check
+	make pycodestyle
 	pylint --rcfile=pylintrc acceptance_tests credentials *.py
 	make quality-js
 
@@ -93,6 +108,9 @@ exec-check_translations_up_to_date: ## test translations on a container
 exec-check_keywords: ## Scan the Django models in all installed apps in this project for restricted field names
 	docker exec -t credentials bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials/ && make check_keywords'
 
+exec-pii_check: ## Check for PII annotations on all Django models
+	docker exec -t credentials bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials/ && make pii_check'
+
 exec-clean: ## Remove all generated files from a container
 	docker exec -t credentials bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials/ && make clean'
 
@@ -111,10 +129,10 @@ exec-tests: ## Run tests on a container
 exec-accept: ## Run acceptance tests on a container
 	docker exec -it credentials bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials/ && make accept'
 
-exec-validate: exec-validate-translations exec-clean exec-static exec-quality exec-tests exec-accept exec-check_keywords ## Run linters and tests after checking translations and gathering static assets
+exec-validate: exec-validate-translations exec-clean exec-static exec-quality exec-tests exec-accept exec-check_keywords exec-pii_check ## Run linters and tests after checking translations and gathering static assets
 
 exec-coverage: ## Generate XML coverage report on a container
-	docker exec -t credentials bash -c 'coverage xml'
+	docker exec -t credentials bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials/ && coverage xml'
 
 html_coverage: ## Generate and view HTML coverage report
 	coverage html && open htmlcov/index.html
@@ -145,8 +163,13 @@ compile_translations: ## Compile translation files, outputting .mo files for eac
 
 fake_translations: extract_translations dummy_translations compile_translations ## Generate and compile dummy translation files
 
+# This Make target should not be removed since it is relied on by a Jenkins job (`edx-internal/tools-edx-jenkins/translation-jobs.yml`), using `ecommerce-scripts/transifex`.
 pull_translations: ## Pull translations from Transifex
-	cd credentials && i18n_tool transifex pull
+	tx pull -af --mode reviewed --minimum-perc=1
+
+# This Make target should not be removed since it is relied on by a Jenkins job (`edx-internal/tools-edx-jenkins/translation-jobs.yml`), using `ecommerce-scripts/transifex`.
+push_translations: ## Push source translation files (.po) to Transifex
+	tx push -s
 
 detect_changed_source_translations: ## Check if translation files are up-to-date
 	cd credentials && i18n_tool changed
@@ -175,3 +198,7 @@ upgrade: piptools ## update the requirements/*.txt files with the latest package
 
 check_keywords: ## Scan the Django models in all installed apps in this project for restricted field names
 	python manage.py check_reserved_keywords --override_file db_keyword_overrides.yml
+
+pii_check: ## Check for PII annotations on all Django models
+	DJANGO_SETTINGS_MODULE=credentials.settings.test \
+	code_annotations django_find_annotations --config_file .pii_annotations.yml --lint --report --coverage

README.rst

@@ -1,17 +1,20 @@
 edX Credentials Service  |Travis|_ |Codecov|_
 =============================================
-.. |Travis| image:: https://travis-ci.org/edx/credentials.svg?branch=master
-.. _Travis: https://travis-ci.org/edx/credentials
+.. |Travis| image:: https://travis-ci.com/edx/credentials.svg?branch=master
+.. _Travis: https://travis-ci.com/edx/credentials
 
 .. |Codecov| image:: http://codecov.io/github/edx/credentials/coverage.svg?branch=master
 .. _Codecov: http://codecov.io/github/edx/credentials?branch=master
 
-This repository contains the edX Credentials Service, used as the backend to support course and program certificates. This service is a replacement for the ``certificates`` app in ``edx-platform``. A more detailed overview is included in the `docs <https://edx-credentials.readthedocs.io/en/latest/overview.html>`_.
+This repository contains the edX Credentials Service, which supports course and program certificates. This service is a replacement for the ``certificates`` app in ``edx-platform``.
 
 Documentation
 -------------
 
-`Documentation <https://edx-credentials.readthedocs.io/en/latest/>`_ is hosted on Read the Docs. The source is hosted in this repo's `docs <https://github.com/edx/credentials/tree/master/docs>`_ directory. To contribute, please open a PR against this repo.
+`Documentation`_ is hosted on Read the Docs. The source is hosted in this repo's `docs`_ directory. To contribute, please open a PR against this repo.
+
+.. _Documentation: https://edx-credentials.readthedocs.io/en/latest/
+.. _docs: https://github.com/edx/credentials/tree/master/docs
 
 License
 -------
@@ -23,7 +26,9 @@ The code in this repository is licensed under version 3 of the AGPL unless other
 How To Contribute
 -----------------
 
-Contributions are welcome. Please read `How To Contribute <https://github.com/edx/edx-platform/blob/master/CONTRIBUTING.rst>`_ for details. Even though it was written with ``edx-platform`` in mind, these guidelines should be followed for Open edX code in general.
+Contributions are welcome. Please read `How To Contribute`_ for details. Even though it was written with ``edx-platform`` in mind, these guidelines should be followed for Open edX code in general.
+
+.. _`How To Contribute`: https://github.com/edx/edx-platform/blob/master/CONTRIBUTING.rst
 
 Reporting Security Issues
 -------------------------
@@ -50,5 +55,9 @@ first, otherwise you'll run into ``webpack_loader.exceptions.WebpackBundleLookup
 Get Help
 --------
 
-Ask questions and discuss this project on `Slack <https://openedx.slack.com/messages/general/>`_ or in the `edx-code Google Group <https://groups.google.com/forum/#!forum/edx-code>`_.
+If you're having trouble, we have `discussion forums`_ where you can connect with others in the community.
+
+Our real-time conversations are on Slack_.
 
+.. _`discussion forums`: https://discuss.openedx.org
+.. _Slack: http://openedx.slack.com/

acceptance_tests/e2e/README.rst

@@ -110,3 +110,4 @@ As discussed above, the acceptance tests rely on configuration which can be spec
 When running against a production-like staging environment, you might run::
 
     $ CREDENTIALS_ROOT_URL="https://credentials.stage.edx.org" LMS_URL_ROOT="https://courses.stage.edx.org" LMS_USERNAME="<username>" LMS_EMAIL="<email address>" LMS_PASSWORD="<password>" ACCESS_TOKEN="<access token>" PROGRAM_UUID=<program_uuid> xvfb-run make accept
+

acceptance_tests/e2e/_test_auth.py

@@ -13,7 +13,7 @@ def setUp(self):
         """
         Instantiate the page objects.
         """
-        super(OAuth2FlowTests, self).setUp()
+        super().setUp()
         self.credentials_api_page = CredentialsDRFPage(self.browser)
 
     def test_login(self):