diff --git a/.github/app_service_config/app_service_config.json b/.github/app_service_config/app_service_config.json deleted file mode 100644 index 129d46a..0000000 --- a/.github/app_service_config/app_service_config.json +++ /dev/null @@ -1,77 +0,0 @@ -[ - { - "name": "CANVAS_ACCOUNT_ID", - "value": "99", - "slotSetting": true - }, - { - "name": "CANVAS_DOMAIN", - "value": "bibsys.instructure.com", - "slotSetting": true - }, - { - "name": "DB_DATABASE", - "value": "canvas-api", - "slotSetting": false - }, - { - "name": "DB_HOST", - "value": "kpas.mysql.database.azure.com", - "slotSetting": true - }, - { - "name": "DJANGO_ALLOWED_HOSTS", - "value": "statistics-api.azurewebsites.net, 127.0.0.1", - "slotSetting": true - }, - { - "name": "DJANGO_DEBUG", - "value": "False", - "slotSetting": true - }, - { - "name": "DOCKER_ENABLE_CI", - "value": "true", - "slotSetting": false - }, - { - "name": "DOCKER_REGISTRY_SERVER_USERNAME", - "value": "udirkpas", - "slotSetting": false - }, - { - "name": "GUNICORN_LOG_LEVEL", - "value": "debug", - "slotSetting": true - }, - { - "name": "GUNICORN_TIMEOUT", - "value": "60", - "slotSetting": true - }, - { - "name": "KPAS_DOMAIN", - "value": "kpas.kompetanse.udir.no", - "slotSetting": true - }, - { - "name": "PULL_MEMBER_COUNTS_FROM_CANVAS_ON_STARTUP", - "value": "False", - "slotSetting": false - }, - { - "name": "WEBSITE_HTTPLOGGING_RETENTION_DAYS", - "value": "2", - "slotSetting": false - }, - { - "name": "WEBSITES_CONTAINER_START_TIME_LIMIT", - "value": "500", - "slotSetting": false - }, - { - "name": "WEBSITES_ENABLE_APP_SERVICE_STORAGE", - "value": "false", - "slotSetting": false - } -] \ No newline at end of file diff --git a/.github/app_service_config/staging_app_service_config.json b/.github/app_service_config/staging_app_service_config.json deleted file mode 100644 index 74b92bf..0000000 --- a/.github/app_service_config/staging_app_service_config.json +++ /dev/null @@ -1,77 +0,0 @@ -[ - { - "name": "CANVAS_ACCOUNT_ID", - "value": "99", - "slotSetting": true - }, - { - "name": "CANVAS_DOMAIN", - "value": "bibsys.instructure.com", - "slotSetting": true - }, - { - "name": "DB_DATABASE", - "value": "canvas-api", - "slotSetting": false - }, - { - "name": "DB_HOST", - "value": "mysql-kpas-staging.mysql.database.azure.com", - "slotSetting": true - }, - { - "name": "DJANGO_ALLOWED_HOSTS", - "value": "statistics-api-staging.azurewebsites.net, 127.0.0.1", - "slotSetting": true - }, - { - "name": "DJANGO_DEBUG", - "value": "False", - "slotSetting": true - }, - { - "name": "DOCKER_ENABLE_CI", - "value": "true", - "slotSetting": false - }, - { - "name": "DOCKER_REGISTRY_SERVER_USERNAME", - "value": "udirkpas", - "slotSetting": false - }, - { - "name": "GUNICORN_LOG_LEVEL", - "value": "debug", - "slotSetting": true - }, - { - "name": "GUNICORN_TIMEOUT", - "value": "60", - "slotSetting": true - }, - { - "name": "KPAS_DOMAIN", - "value": "kpas.staging.kompetanse.udir.no", - "slotSetting": true - }, - { - "name": "PULL_MEMBER_COUNTS_FROM_CANVAS_ON_STARTUP", - "value": "False", - "slotSetting": false - }, - { - "name": "WEBSITE_HTTPLOGGING_RETENTION_DAYS", - "value": "2", - "slotSetting": false - }, - { - "name": "WEBSITES_CONTAINER_START_TIME_LIMIT", - "value": "500", - "slotSetting": false - }, - { - "name": "WEBSITES_ENABLE_APP_SERVICE_STORAGE", - "value": "false", - "slotSetting": false - } -] \ No newline at end of file diff --git a/.github/workflows/health-test.yml b/.github/workflows/health-test.yml index 10a63c7..77c6f1d 100644 --- a/.github/workflows/health-test.yml +++ b/.github/workflows/health-test.yml @@ -21,7 +21,7 @@ jobs: steps: # GitHub repository checkout - name: GitHub repository checkout - uses: actions/checkout@v1 + uses: actions/checkout@v4 - name: Check if data from API is less than DATA_MAX_AGE seconds old diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 664f45a..495ef16 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,13 +20,7 @@ jobs: runs-on: ubuntu-latest env: DB_HOST: 127.0.0.1 - KPAS_DOMAIN: kpas-lti-staging-kpas.azurewebsites.net #NB! must match certificate in .github/kpas_ssl_certs - CANVAS_DOMAIN: bibsys.instructure.com - CANVAS_ACCOUNT_ID: 99 - KPAS_LTI_GIT_COMMIT: 7274efb45554b025d3b9800ab4e915003fe81fb0 #Exactly which state in the KPAS-LTI git repository should be used as dependency during our tests DJANGO_SECRET_KEY: not_secret_key - #CA_FILE_NAME: ca.crt - AZURE_CONTAINER_REGISTRY: udirkpas.azurecr.io AZURE_RESOURCE_GROUP: laravel AZURE_APP_NAME: statistics-api @@ -35,7 +29,7 @@ jobs: steps: # GitHub repository checkout - name: GitHub repository checkout - uses: actions/checkout@v1 + uses: actions/checkout@v4 - name: Log in to Azure run: | @@ -65,7 +59,7 @@ jobs: - name: Run Django unit tests run: | - DJANGO_DEBUG=True CANVAS_ACCESS_KEY=${{ secrets.CANVAS_ACCESS_KEY }} DJANGO_SETTINGS_MODULE=statistics_api.tests.test_settings DB_DATABASE=canvas-api DB_USERNAME=root DB_PASSWORD=root DB_PORT=3307 PYTHONPATH="${GITHUB_WORKSPACE}" python3 manage.py test --settings statistics_api.tests.test_settings --verbosity=3 + DJANGO_DEBUG=True DJANGO_SETTINGS_MODULE=statistics_api.tests.test_settings DB_DATABASE=canvas-api DB_USERNAME=root DB_PASSWORD=root DB_PORT=3307 PYTHONPATH="${GITHUB_WORKSPACE}" python3 manage.py test --settings statistics_api.tests.test_settings --verbosity=3 # Checking that only the issue is the SECURE_HSTS_SECONDS warning - name: Run manage.py check --deploy @@ -83,33 +77,13 @@ jobs: if [ "$GITHUB_REF" = "refs/heads/staging" ] then docker build --build-arg WEBSITES_PORT=$WEBSITES_PORT . -f Dockerfile -t $AZURE_CONTAINER_REGISTRY/statistics-api-staging:$GITHUB_SHA -t $AZURE_CONTAINER_REGISTRY/statistics-api-staging:latest - az webapp config appsettings set --subscription ${{ secrets.AZURE_SUBSCRIPTION_ID }} -g $AZURE_RESOURCE_GROUP -n $AZURE_APP_NAME --slot staging --settings \ - "CANVAS_ACCESS_KEY=${{ secrets.CANVAS_ACCESS_KEY }}" \ - "DB_PASSWORD=${{ secrets.STAGING_DB_PASSWORD }}" \ - "DB_USERNAME=${{ secrets.STAGING_DB_USERNAME }}" \ - "DJANGO_SECRET_KEY=${{ secrets.STAGING_DJANGO_SECRET_KEY }}" \ - "DOCKER_REGISTRY_SERVER_PASSWORD=${{ secrets.DOCKER_REGISTRY_SERVER_PASSWORD }}" \ - "KPAS_API_ACCESS_TOKEN=${{ secrets.STAGING_KPAS_API_ACCESS_TOKEN }}" \ - "WEBSITES_PORT=$WEBSITES_PORT" \ - "DOCKER_REGISTRY_SERVER_URL=https://$AZURE_CONTAINER_REGISTRY" \ - "GIT_COMMIT=${GITHUB_SHA}" \ - @"${GITHUB_WORKSPACE}/.github/app_service_config/staging_app_service_config.json" + az webapp config appsettings set --subscription ${{ secrets.AZURE_SUBSCRIPTION_ID }} -g $AZURE_RESOURCE_GROUP -n $AZURE_APP_NAME --slot staging --settings "GIT_COMMIT=${GITHUB_SHA}" docker push $AZURE_CONTAINER_REGISTRY/statistics-api-staging:$GITHUB_SHA docker push $AZURE_CONTAINER_REGISTRY/statistics-api-staging:latest elif [ "$GITHUB_REF" = "refs/heads/master" ] then docker build --build-arg WEBSITES_PORT=$WEBSITES_PORT . -f Dockerfile -t $AZURE_CONTAINER_REGISTRY/statistics-api:$GITHUB_SHA -t $AZURE_CONTAINER_REGISTRY/statistics-api:latest - az webapp config appsettings set --subscription ${{ secrets.AZURE_SUBSCRIPTION_ID }} -g $AZURE_RESOURCE_GROUP -n $AZURE_APP_NAME --settings \ - "CANVAS_ACCESS_KEY=${{ secrets.CANVAS_ACCESS_KEY }}" \ - "DB_PASSWORD=${{ secrets.DB_PASSWORD }}" \ - "DB_USERNAME=${{ secrets.DB_USERNAME }}" \ - "DJANGO_SECRET_KEY=${{ secrets.DJANGO_SECRET_KEY }}" \ - "DOCKER_REGISTRY_SERVER_PASSWORD=${{ secrets.DOCKER_REGISTRY_SERVER_PASSWORD }}" \ - "KPAS_API_ACCESS_TOKEN=${{ secrets.KPAS_API_ACCESS_TOKEN }}" \ - "WEBSITES_PORT=$WEBSITES_PORT" \ - "DOCKER_REGISTRY_SERVER_URL=https://$AZURE_CONTAINER_REGISTRY" \ - "GIT_COMMIT=${GITHUB_SHA}" \ - @"${GITHUB_WORKSPACE}/.github/app_service_config/app_service_config.json" + az webapp config appsettings set --subscription ${{ secrets.AZURE_SUBSCRIPTION_ID }} -g $AZURE_RESOURCE_GROUP -n $AZURE_APP_NAME --settings "GIT_COMMIT=${GITHUB_SHA}" docker push $AZURE_CONTAINER_REGISTRY/statistics-api:$GITHUB_SHA docker push $AZURE_CONTAINER_REGISTRY/statistics-api:latest fi diff --git a/README.md b/README.md index 3f0cdb9..039324a 100644 --- a/README.md +++ b/README.md @@ -3,33 +3,73 @@ Python Django application which serves HTTP endpoints that provide statistical a # Setup -This app requires a local or remote instance of Canvas LMS and an instance of KPAS LTI (https://github.com/matematikk-mooc/kpas-api), from which this service retrieves data about schools, counties, municipalities and associations thereof. You will need an API key to Canvas LMS which is valid for some Canvas account ID. Copy `.env.example` to new file `.env`, and fill in the domain of your Canvas LMS instance and the API key of your root account, and alter other attributes, if necessary. Adjust settings in `docker-compose.dev.yml` for your environment, if necessary. Run +This app requires an instance of Canvas LMS, from which this service retrieves data about schools, counties, municipalities and associations thereof. You will need an API key to Canvas LMS which is valid for some Canvas account ID. Use file `.env.dev` to update enviroment variables when running locally, and fill in the domain of your Canvas LMS instance and the API key of your root account, and alter other attributes, if necessary. When running locally the Canvas LMS instance needs to be set to the Canvas test environment. -`docker-compose -f docker-compose.dev.yml up` +Adjust settings in `docker-compose.dev.yml` for your environment, if necessary. +To start the application locally do the following steps: -The application is hosted at `statistics-api-dev.local:8000` by default. Import the CA certificate `ca.crt` to your web browser to enable HTTPS. Add new line to `/etc/hosts`: `127.0.0.1 statistics-api-dev.local`, routing the domain `statistics-api-dev.local` to your host IP. If you're not running a Linux OS, the procedure to add new domain route will be somewhat different. +- Activate venv: `source ./venv/bin/activate` +- Run `run_development.sh` in venv. +- Migrate the db: `python manage.py migrate` +- Populate the db by running the commands below +- Access api at `http://localhost:8000/api` +- Available endpoints is specified in urls.py -You may also create your own CA certificate and replace `ca.crt` and create new site certificate and private key in `nginx/nginx-selfsigned.crt` and `nginx/nginx-selfsigned.key` respectively. Creating your own CA would mitigate the small risk of an attacker impersonating the `statistics-api-dev.local` domain. +Access application on e.g. `http://127.0.0.1:8000/api/statistics/:courseId` -Access application on e.g. `https://statistics-api-dev.local:8000/api/statistics/:courseId` +### Commands to populate db with statistics -# Testing +To populatet the db run `python manage.py ` + +#### import_county_teacher_counts_from_csv + +This command will populate the db with high school teacher counts for each county. This statistics is found in a csv file located in the `data` folder. +This file needs to be updated yearly. + +#### import_school_teacher_counts_from_csv + +This command will populate the db with primary school teacher counts for each school. This statistics is found in a csv file located in the `data` folder. +This file needs to be updated yearly. + +#### fetch_course_enrollment_activity + +This command will populate the db with number of enrolled users who has been active the last 24 hours + +#### pull_course_group_registrations + +This command will populate the db with number of new group and course registrations the last day + + +#### pull_data_from_matomo + +This populates the db with visit and page statistics from matomo. To run this command you will need to set the MATOMO_ACCESS_KEY in the `.env.dev` file. -There are a number of tests in this repository, but nearly all of them are integration tests dependent on 3rd party service KPAS LTI. You will need to configure environment variables to a running instance of KPAS LTI. statistics-api does not mutate the state in KPAST LTI, so using a remote instance should be safe. +#### pull_finnish_marks_canvas -A working test environment is automatically built in GitHub Actions pipelines. Any time you push to the `test` branch, all unit tests will be run. The pipeline in GitHub actions builds an instance of KPAS LTI using Docker and docker-compose. +This populates the db with statistics on finnsih marks for each module item. + +#### pull_history_from_canvas_and_update_db + +Populates the db with history statisics from canvas + + +#### pull_total_students_counts_from_canvas + +Populates the db with number of studens in groups from canvas. + + + + +# Testing + +There are some unit tests located in the folder 'tests'. These will run when merging changes into the branches `staging` and `master`, as a part of the depoloyment. If any of the tests fails the deployment will stop. # Deployment -Any merge to `staging` or `master` branch will automatically deploy the application to the staging and production environments respectively. Unit tests will also be run, and deployment will stop if any test fails. +Any merge to `staging` or `master` branch will automatically deploy the application to the staging and production environments respectively. # Documentation Swagger UI documentation is available at https://matematikk-mooc.github.io/statistics-api-documentation/, using GitHub pages. The source for the documentation is available at https://github.com/matematikk-mooc/statistics-api-documentation. - -# Run application locally with local KPAS -- Add canvas and matomo keys to `.env.dev` -- Run `run_development.sh` in venv. -- Access api at `http://localhost:8000/api` \ No newline at end of file diff --git a/nginx/default.conf b/nginx/default.conf index 9961c56..94378a0 100644 --- a/nginx/default.conf +++ b/nginx/default.conf @@ -1,6 +1,6 @@ server { - server_name NGINX_SERVER_NAME ; + server_name statapi ; listen 443 ssl default_server; listen [::]:443 ssl default_server; ssl_certificate /etc/ssl/certs/nginx-selfsigned.crt; diff --git a/requirements.txt b/requirements.txt index 0f6e164..762be5a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,5 @@ mysqlclient==2.1.1 arrow~=1.3.0 python-graphql-client~=0.4.3 djangorestframework>=3.14 -bugsnag \ No newline at end of file +bugsnag +netifaces diff --git a/statistics_api/definitions.py b/statistics_api/definitions.py index 2f9014c..07960c1 100644 --- a/statistics_api/definitions.py +++ b/statistics_api/definitions.py @@ -1,6 +1,6 @@ import os from distutils import util - +import netifaces as ni from sqlalchemy.ext.declarative import declarative_base #### START OF ENVIRONMENT VARIABLES #### @@ -37,8 +37,16 @@ DJANGO_SECRET_KEY = os.getenv("DJANGO_SECRET_KEY") DJANGO_DEBUG = bool(util.strtobool(os.getenv("DJANGO_DEBUG"))) if os.getenv("DJANGO_DEBUG") is not None else False -DJANGO_ALLOWED_HOSTS = [s.strip() for s in os.getenv("DJANGO_ALLOWED_HOSTS").split(',')] if os.getenv( - "DJANGO_ALLOWED_HOSTS") else ["*"] +allowed_hosts = [s.strip() for s in os.getenv("DJANGO_ALLOWED_HOSTS", "").split(',')] +if not allowed_hosts: + allowed_hosts = ["*"] + +ni.ifaddresses('eth0') +ip = ni.ifaddresses('eth0')[ni.AF_INET][0]['addr'] + +allowed_hosts.append(ip) + +DJANGO_ALLOWED_HOSTS = allowed_hosts BUGSNAG_API_KEY = os.getenv("BUGSNAG_API_KEY") diff --git a/statistics_api/settings.py b/statistics_api/settings.py index 73ea227..fb93ccd 100644 --- a/statistics_api/settings.py +++ b/statistics_api/settings.py @@ -158,6 +158,7 @@ SECURE_REFERRER_POLICY = 'strict-origin-when-cross-origin' CORS_ORIGIN_ALLOW_ALL = True + # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/1.11/howto/static-files/