Staging

.github/workflows/health-test.yml

@@ -21,7 +21,7 @@ jobs:
     steps:
       # GitHub repository checkout
       - name: GitHub repository checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v4
 
 
       - name: Check if data from API is less than DATA_MAX_AGE seconds old

.github/workflows/tests.yml

@@ -20,13 +20,7 @@ jobs:
     runs-on: ubuntu-latest
     env:
       DB_HOST: 127.0.0.1
-      KPAS_DOMAIN: kpas-lti-staging-kpas.azurewebsites.net  #NB! must match certificate in .github/kpas_ssl_certs
-      CANVAS_DOMAIN: bibsys.instructure.com
-      CANVAS_ACCOUNT_ID: 99
-      KPAS_LTI_GIT_COMMIT: 7274efb45554b025d3b9800ab4e915003fe81fb0 #Exactly which state in the KPAS-LTI git repository should be used as dependency during our tests
       DJANGO_SECRET_KEY: not_secret_key
-      #CA_FILE_NAME: ca.crt
-
       AZURE_CONTAINER_REGISTRY: udirkpas.azurecr.io
       AZURE_RESOURCE_GROUP: laravel
       AZURE_APP_NAME: statistics-api
@@ -35,7 +29,7 @@ jobs:
     steps:
         # GitHub repository checkout
       - name: GitHub repository checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v4
 
       - name: Log in to Azure
         run: |
@@ -65,7 +59,7 @@ jobs:
 
       - name: Run Django unit tests
         run: |
-          DJANGO_DEBUG=True CANVAS_ACCESS_KEY=${{ secrets.CANVAS_ACCESS_KEY }} DJANGO_SETTINGS_MODULE=statistics_api.tests.test_settings DB_DATABASE=canvas-api DB_USERNAME=root DB_PASSWORD=root DB_PORT=3307 PYTHONPATH="${GITHUB_WORKSPACE}" python3 manage.py test --settings statistics_api.tests.test_settings --verbosity=3
+          DJANGO_DEBUG=True DJANGO_SETTINGS_MODULE=statistics_api.tests.test_settings DB_DATABASE=canvas-api DB_USERNAME=root DB_PASSWORD=root DB_PORT=3307 PYTHONPATH="${GITHUB_WORKSPACE}" python3 manage.py test --settings statistics_api.tests.test_settings --verbosity=3
 
       # Checking that only the issue is the SECURE_HSTS_SECONDS warning
       - name: Run manage.py check --deploy
@@ -83,33 +77,13 @@ jobs:
           if [ "$GITHUB_REF" = "refs/heads/staging" ]
           then
             docker build --build-arg WEBSITES_PORT=$WEBSITES_PORT . -f Dockerfile -t $AZURE_CONTAINER_REGISTRY/statistics-api-staging:$GITHUB_SHA -t $AZURE_CONTAINER_REGISTRY/statistics-api-staging:latest
-            az webapp config appsettings set --subscription ${{ secrets.AZURE_SUBSCRIPTION_ID }} -g $AZURE_RESOURCE_GROUP -n $AZURE_APP_NAME --slot staging --settings \
-              "CANVAS_ACCESS_KEY=${{ secrets.CANVAS_ACCESS_KEY }}" \
-              "DB_PASSWORD=${{ secrets.STAGING_DB_PASSWORD }}" \
-              "DB_USERNAME=${{ secrets.STAGING_DB_USERNAME }}" \
-              "DJANGO_SECRET_KEY=${{ secrets.STAGING_DJANGO_SECRET_KEY }}" \
-              "DOCKER_REGISTRY_SERVER_PASSWORD=${{ secrets.DOCKER_REGISTRY_SERVER_PASSWORD }}" \
-              "KPAS_API_ACCESS_TOKEN=${{ secrets.STAGING_KPAS_API_ACCESS_TOKEN }}" \
-              "WEBSITES_PORT=$WEBSITES_PORT" \
-              "DOCKER_REGISTRY_SERVER_URL=https://$AZURE_CONTAINER_REGISTRY" \
-              "GIT_COMMIT=${GITHUB_SHA}" \
-              @"${GITHUB_WORKSPACE}/.github/app_service_config/staging_app_service_config.json"
+            az webapp config appsettings set --subscription ${{ secrets.AZURE_SUBSCRIPTION_ID }} -g $AZURE_RESOURCE_GROUP -n $AZURE_APP_NAME --slot staging --settings "GIT_COMMIT=${GITHUB_SHA}"
             docker push $AZURE_CONTAINER_REGISTRY/statistics-api-staging:$GITHUB_SHA
             docker push $AZURE_CONTAINER_REGISTRY/statistics-api-staging:latest
           elif [ "$GITHUB_REF" = "refs/heads/master" ]
           then
             docker build --build-arg WEBSITES_PORT=$WEBSITES_PORT . -f Dockerfile -t $AZURE_CONTAINER_REGISTRY/statistics-api:$GITHUB_SHA -t $AZURE_CONTAINER_REGISTRY/statistics-api:latest
-            az webapp config appsettings set --subscription ${{ secrets.AZURE_SUBSCRIPTION_ID }} -g $AZURE_RESOURCE_GROUP -n $AZURE_APP_NAME --settings \
-              "CANVAS_ACCESS_KEY=${{ secrets.CANVAS_ACCESS_KEY }}" \
-              "DB_PASSWORD=${{ secrets.DB_PASSWORD }}" \
-              "DB_USERNAME=${{ secrets.DB_USERNAME }}" \
-              "DJANGO_SECRET_KEY=${{ secrets.DJANGO_SECRET_KEY }}" \
-              "DOCKER_REGISTRY_SERVER_PASSWORD=${{ secrets.DOCKER_REGISTRY_SERVER_PASSWORD }}" \
-              "KPAS_API_ACCESS_TOKEN=${{ secrets.KPAS_API_ACCESS_TOKEN }}" \
-              "WEBSITES_PORT=$WEBSITES_PORT" \
-              "DOCKER_REGISTRY_SERVER_URL=https://$AZURE_CONTAINER_REGISTRY" \
-              "GIT_COMMIT=${GITHUB_SHA}" \
-              @"${GITHUB_WORKSPACE}/.github/app_service_config/app_service_config.json"
+            az webapp config appsettings set --subscription ${{ secrets.AZURE_SUBSCRIPTION_ID }} -g $AZURE_RESOURCE_GROUP -n $AZURE_APP_NAME --settings "GIT_COMMIT=${GITHUB_SHA}"
             docker push $AZURE_CONTAINER_REGISTRY/statistics-api:$GITHUB_SHA
             docker push $AZURE_CONTAINER_REGISTRY/statistics-api:latest
             fi

README.md

@@ -3,33 +3,73 @@ Python Django application which serves HTTP endpoints that provide statistical a
 
 # Setup
 
-This app requires a local or remote instance of Canvas LMS and an instance of KPAS LTI (https://github.com/matematikk-mooc/kpas-api), from which this service retrieves data about schools, counties, municipalities and associations thereof. You will need an API key to Canvas LMS which is valid for some Canvas account ID. Copy `.env.example` to new file `.env`, and fill in the domain of your Canvas LMS instance and the API key of your root account, and alter other attributes, if necessary. Adjust settings in `docker-compose.dev.yml` for your environment, if necessary. Run 
+This app requires an instance of Canvas LMS, from which this service retrieves data about schools, counties, municipalities and associations thereof. You will need an API key to Canvas LMS which is valid for some Canvas account ID. Use file `.env.dev` to update enviroment variables when running locally, and fill in the domain of your Canvas LMS instance and the API key of your root account, and alter other attributes, if necessary. When running locally the Canvas LMS instance needs to be set to the Canvas test environment.
 
-`docker-compose -f docker-compose.dev.yml up`
+Adjust settings in `docker-compose.dev.yml` for your environment, if necessary.
+To start the application locally do the following steps:
 
-The application is hosted at `statistics-api-dev.local:8000` by default. Import the CA certificate `ca.crt` to your web browser to enable HTTPS. Add new line to `/etc/hosts`: `127.0.0.1 statistics-api-dev.local`, routing the domain `statistics-api-dev.local` to your host IP. If you're not running a Linux OS, the procedure to add new domain route will be somewhat different.
+- Activate venv: `source ./venv/bin/activate`
+- Run `run_development.sh` in venv.
+- Migrate the db: `python manage.py migrate`
+- Populate the db by running the commands below
+- Access api at `http://localhost:8000/api`
+- Available endpoints is specified in urls.py
 
-You may also create your own CA certificate and replace `ca.crt` and create new site certificate and private key in `nginx/nginx-selfsigned.crt` and `nginx/nginx-selfsigned.key` respectively. Creating your own CA would mitigate the small risk of an attacker impersonating the `statistics-api-dev.local` domain.
+Access application on e.g. `http://127.0.0.1:8000/api/statistics/:courseId`
 
-Access application on e.g. `https://statistics-api-dev.local:8000/api/statistics/:courseId`
 
+### Commands to populate db with statistics
 
-# Testing
+To populatet the db run `python manage.py <name of command>`
+
+#### import_county_teacher_counts_from_csv
+
+This command will populate the db with high school teacher counts for each county. This statistics is found in a csv file located in the `data` folder.
+This file needs to be updated yearly.
+
+#### import_school_teacher_counts_from_csv
+
+This command will populate the db with primary school teacher counts for each school. This statistics is found in a csv file located in the `data` folder.
+This file needs to be updated yearly.
+
+#### fetch_course_enrollment_activity
+
+This command will populate the db with number of enrolled users who has been active the last 24 hours
+
+#### pull_course_group_registrations
+
+This command will populate the db with number of new group and course registrations the last day
+
+
+#### pull_data_from_matomo
+
+This populates the db with visit and page statistics from matomo. To run this command you will need to set the MATOMO_ACCESS_KEY in the `.env.dev` file.
 
-There are a number of tests in this repository, but nearly all of them are integration tests dependent on 3rd party service KPAS LTI. You will need to configure environment variables to a running instance of KPAS LTI. statistics-api does not mutate the state in KPAST LTI, so using a remote instance should be safe.
+#### pull_finnish_marks_canvas
 
-A working test environment is automatically built in GitHub Actions pipelines. Any time you push to the `test` branch, all unit tests will be run. The pipeline in GitHub actions builds an instance of KPAS LTI using Docker and docker-compose.
+This populates the db with statistics on finnsih marks for each module item.
+
+#### pull_history_from_canvas_and_update_db
+
+Populates the db with history statisics from canvas
+
+
+#### pull_total_students_counts_from_canvas
+
+Populates the db with number of studens in groups from canvas.
+
+
+
+
+# Testing
+
+There are some unit tests located in the folder 'tests'. These will run when merging changes into the branches `staging` and `master`, as a part of the depoloyment. If any of the tests fails the deployment will stop.
 
 
 # Deployment
 
-Any merge to `staging` or `master` branch will automatically deploy the application to the staging and production environments respectively. Unit tests will also be run, and deployment will stop if any test fails.
+Any merge to `staging` or `master` branch will automatically deploy the application to the staging and production environments respectively.
 
 # Documentation
 
 Swagger UI documentation is available at https://matematikk-mooc.github.io/statistics-api-documentation/, using GitHub pages. The source for the documentation is available at https://github.com/matematikk-mooc/statistics-api-documentation.
-
-# Run application locally with local KPAS
-- Add canvas and matomo keys to `.env.dev` 
-- Run `run_development.sh` in venv. 
-- Access api at `http://localhost:8000/api`

nginx/default.conf

@@ -1,6 +1,6 @@
 server {
 
-    server_name NGINX_SERVER_NAME	;
+    server_name statapi	;
     listen 443 ssl default_server;
     listen [::]:443 ssl default_server;
     ssl_certificate /etc/ssl/certs/nginx-selfsigned.crt;

requirements.txt

@@ -10,4 +10,5 @@ mysqlclient==2.1.1
 arrow~=1.3.0
 python-graphql-client~=0.4.3
 djangorestframework>=3.14
-bugsnag
+bugsnag
+netifaces

statistics_api/definitions.py

@@ -1,6 +1,6 @@
 import os
 from distutils import util
-
+import netifaces as ni
 from sqlalchemy.ext.declarative import declarative_base
 
 ####    START OF ENVIRONMENT VARIABLES  ####
@@ -37,8 +37,16 @@
 
 DJANGO_SECRET_KEY = os.getenv("DJANGO_SECRET_KEY")
 DJANGO_DEBUG = bool(util.strtobool(os.getenv("DJANGO_DEBUG"))) if os.getenv("DJANGO_DEBUG") is not None else False
-DJANGO_ALLOWED_HOSTS = [s.strip() for s in os.getenv("DJANGO_ALLOWED_HOSTS").split(',')] if os.getenv(
-    "DJANGO_ALLOWED_HOSTS") else ["*"]
+allowed_hosts = [s.strip() for s in os.getenv("DJANGO_ALLOWED_HOSTS", "").split(',')]
+if not allowed_hosts:
+    allowed_hosts = ["*"]
+
+ni.ifaddresses('eth0')
+ip = ni.ifaddresses('eth0')[ni.AF_INET][0]['addr']
+
+allowed_hosts.append(ip)
+
+DJANGO_ALLOWED_HOSTS = allowed_hosts
 
 BUGSNAG_API_KEY = os.getenv("BUGSNAG_API_KEY")
 

statistics_api/settings.py

@@ -158,6 +158,7 @@
 SECURE_REFERRER_POLICY = 'strict-origin-when-cross-origin'
 CORS_ORIGIN_ALLOW_ALL = True
 
+
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/1.11/howto/static-files/