Add new fields for wordpress filters EDLY-6765

.coveragerc

@@ -7,6 +7,8 @@ omit =
     course_discovery/settings*
     course_discovery/conf*
     course_discovery/apps/course_metadata/tests/factories.py
+    course_discovery/apps/course_metadata/algolia_models.py
+    course_discovery/apps/course_metadata/index.py
     *conftest.py
     *wsgi.py
     *migrations*

.github/workflows/docker-publish.yml

@@ -0,0 +1,22 @@
+name: Push Docker Images
+
+on:
+  push:
+    branches:
+      - master
+jobs:
+  # Push image to GitHub Packages.
+  # See also https://docs.docker.com/docker-hub/builds/
+  push:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push'
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Build and Push docker image
+        env:
+          DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
+          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+        run : make docker_push

.github/workflows/semgrep-lint.yml

@@ -0,0 +1,17 @@
+name: Semgrep
+on: [pull_request]
+jobs:
+    semgrep:
+        runs-on: ubuntu-latest
+        continue-on-error: true
+        name: Check
+        steps:
+            - uses: actions/checkout@v1
+            - uses: returntocorp/semgrep-action@v1
+              name: django rules
+              with:
+                config: p/django
+            - uses: returntocorp/semgrep-action@v1
+              name: other rules
+              with:
+                config: https://semgrep.live/dlukeomalley:use-assertEqual-for-equality

.pep8 → .pycodestyle

@@ -1,4 +1,4 @@
-[pep8]
-ignore=E501
+[pycodestyle]
+ignore=E501,W504
 max_line_length=119
 exclude=settings,migrations,course_discovery/static,bower_components,course_discovery/wsgi.py

.travis.yml

@@ -1,7 +1,8 @@
+dist: focal
 language: python
 
 python:
-    - "3.5"
+  - 3.8
 
 branches:
   only:
@@ -14,43 +15,41 @@ sudo: required
 
 cache:
   - directories:
-    - node_modules
     - course_discovery/static/bower_components
 
 before_install:
     - make travis_up
 
 matrix:
   include:
+    - env: TOXENV=django22
+    - env: TOXENV=django30
     - env: COMMAND=test:quality
-
-      install:
-        - docker exec -t discovery bash -c 'sed -i "s/course_discovery.settings.devstack/course_discovery.settings.test/" /edx/app/discovery/discovery_env'
-        - docker exec -t discovery bash -c 'source /edx/app/discovery/discovery_env && cd /edx/app/discovery/discovery/ && make requirements'
-
-      script:
-          - docker exec -t discovery bash -c 'source /edx/app/discovery/discovery_env && cd /edx/app/discovery/discovery/ && make docs'
-          - docker exec -t discovery bash -c 'source /edx/app/discovery/discovery_env && cd /edx/app/discovery/discovery/ && make check_translations_up_to_date'
-          - docker exec -t discovery bash -c 'source /edx/app/discovery/discovery_env && cd /edx/app/discovery/discovery/ && make validate_translations'
-          - docker exec -t discovery bash -c 'source /edx/app/discovery/discovery_env && cd /edx/app/discovery/discovery/ && make clean_static'
-          - docker exec -t discovery bash -c 'source /edx/app/discovery/discovery_env && cd /edx/app/discovery/discovery/ && make static'
-          - docker exec -t discovery bash -c 'source /edx/app/discovery/discovery_env && cd /edx/app/discovery/discovery/ && make quality'
-
-    - env: COMMAND=test:unittests
-
       install:
-        - docker exec -t discovery bash -c 'apt update && apt install -y xvfb firefox gettext wget'
-        # Remove firefox but leave its dependencies, and then download and install a working version of firefox.
-        - docker exec -t discovery bash -c 'sudo dpkg -r --force-all firefox && TEMP_DEB="$(mktemp)" && wget -O "$TEMP_DEB" https://s3.amazonaws.com/vagrant.testeng.edx.org/firefox_61.0.1%2Bbuild1-0ubuntu0.16.04.1_amd64.deb && dpkg -i "$TEMP_DEB"'
         - docker exec -t discovery bash -c 'sed -i "s/course_discovery.settings.devstack/course_discovery.settings.test/" /edx/app/discovery/discovery_env'
         - docker exec -t discovery bash -c 'source /edx/app/discovery/discovery_env && cd /edx/app/discovery/discovery/ && make requirements'
 
       script:
-          - docker exec -t discovery bash -c 'source /edx/app/discovery/discovery_env && cd /edx/app/discovery/discovery/ && make clean_static'
-          - docker exec -t discovery bash -c 'source /edx/app/discovery/discovery_env && cd /edx/app/discovery/discovery/ && make static'
-          - docker exec -t discovery bash -c 'source /edx/app/discovery/discovery_env && cd /edx/app/discovery/discovery/ && xvfb-run make test'
-
-      after_success:
-          - pip install -U codecov
-          - docker exec -t discovery bash -c 'source /edx/app/discovery/discovery_env && cd /edx/app/discovery/discovery/ && coverage xml'
-          - codecov
+        - docker exec -t discovery bash -c 'source /edx/app/discovery/discovery_env && cd /edx/app/discovery/discovery/ && make docs'
+        - docker exec -t discovery bash -c 'source /edx/app/discovery/discovery_env && cd /edx/app/discovery/discovery/ && make clean_static'
+        - docker exec -t discovery bash -c 'source /edx/app/discovery/discovery_env && cd /edx/app/discovery/discovery/ && make static'
+        - docker exec -t discovery bash -c 'source /edx/app/discovery/discovery_env && cd /edx/app/discovery/discovery/ && make quality'
+        - docker exec -t discovery bash -c 'source /edx/app/discovery/discovery_env && cd /edx/app/discovery/discovery/ && make check_keywords'
+  allow_failures:
+    - env: TOXENV=django30
+
+install:
+  - docker exec -t discovery bash -c 'sed -i "s|http://archive|http://us.archive|g" /etc/apt/sources.list' # US mirrors for speed
+  - docker exec -t discovery bash -c 'apt update && apt install -y --no-install-recommends firefox gettext'
+  - docker exec -t discovery bash -c 'sed -i "s/course_discovery.settings.devstack/course_discovery.settings.test/" /edx/app/discovery/discovery_env'
+  - docker exec -e TOXENV=$TOXENV -t discovery bash -c 'source /edx/app/discovery/discovery_env && cd /edx/app/discovery/discovery/ && make requirements'
+
+script:
+  - docker exec -t discovery bash -c 'source /edx/app/discovery/discovery_env && cd /edx/app/discovery/discovery/ && make clean_static'
+  - docker exec -t discovery bash -c 'source /edx/app/discovery/discovery_env && cd /edx/app/discovery/discovery/ && make static'
+  - docker exec -e TOXENV=$TOXENV -t discovery bash -c 'source /edx/app/discovery/discovery_env && cd /edx/app/discovery/discovery/ && make test'
+
+after_success:
+  - pip install -U codecov
+  - docker exec -t discovery bash -c 'source /edx/app/discovery/discovery_env && cd /edx/app/discovery/discovery/ && coverage xml'
+  - codecov

.travis/docker-compose-travis.yml

@@ -2,20 +2,17 @@ version: "2"
 
 services:
   db:
-    image: mysql:5.6
+    image: mysql:5.7
     container_name: db
     command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci
     environment:
-      MYSQL_ROOT_PASSWORD: ""
-      MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
-      MYSQL_USER: "discov001"
-      MYSQL_PASSWORD: "password"
+      MYSQL_ROOT_PASSWORD: "password"
       MYSQL_DATABASE: "discovery"
   es:
-    image: elasticsearch:1.5.2
+    image: elasticsearch:1.7.6
     container_name: es
   memcached:
-    image: memcached:1.4.24
+    image: memcached:1.6.8
     container_name: memcached
   discovery:
     # Uncomment this line to use the official course-discovery base image
@@ -44,7 +41,7 @@ services:
       DB_NAME: "discovery"
       DB_PASSWORD: "password"
       DB_PORT: "3306"
-      DB_USER: "discov001"
+      DB_USER: "root"
       DJANGO_SETTINGS_MODULE: "course_discovery.settings.test"
       ENABLE_DJANGO_TOOLBAR: 1
       TEST_ELASTICSEARCH_URL: "http://es:9200"

.travis/run_tests.sh

@@ -3,7 +3,7 @@
 . /edx/app/discovery/nodeenvs/discovery/bin/activate
 
 apt update
-apt install -y xvfb firefox gettext wget
+apt install -y xvfb firefox gettext
 
 cd /edx/app/discovery/discovery
 export PATH=$PATH:$PWD/node_modules/.bin
@@ -18,9 +18,6 @@ make requirements.js
 # Ensure documentation can be compiled
 make docs
 
-# Check if translation files are up-to-date
-make validate_translations
-
 # Compile assets and run validation
 make clean_static
 make static

CODEOWNERS

@@ -0,0 +1,2 @@
+# The following users are the owners of all course-discovery files
+*       @edx/course-discovery-admins

Dockerfile

@@ -0,0 +1,72 @@
+FROM ubuntu:focal as app
+
+# System requirements.
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && \
+	apt-get upgrade -qy
+RUN apt-get install --yes \
+	git \
+	language-pack-en \
+	python3-venv \
+	python3.8-dev \
+	python3.8-venv \
+	build-essential \
+	libffi-dev \
+	libmysqlclient-dev \
+	libxml2-dev \
+	libxslt1-dev \
+	libjpeg-dev \
+	libssl-dev
+
+RUN rm -rf /var/lib/apt/lists/*
+
+ENV VIRTUAL_ENV=/venv
+RUN python3.8 -m venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
+RUN pip install pip==20.2.3 setuptools==50.3.0 nodeenv
+
+# Use UTF-8.
+RUN locale-gen en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
+
+# Make necessary directories and environment variables.
+RUN mkdir -p /edx/var/discovery/staticfiles
+RUN mkdir -p /edx/var/discovery/media
+ENV DJANGO_SETTINGS_MODULE course_discovery.settings.production
+
+# Working directory will be root of repo.
+WORKDIR /edx/app/discovery
+
+# Copy just JS requirements and install them.
+COPY package.json package.json
+COPY package-lock.json package-lock.json
+RUN nodeenv /edx/app/nodeenv --node=12.11.1 --prebuilt
+ENV PATH /edx/app/nodeenv/bin:${PATH}
+RUN npm install --production
+COPY bower.json bower.json
+RUN ./node_modules/.bin/bower install --allow-root --production
+
+# Copy just Python requirements & install them.
+COPY requirements/ requirements/
+RUN pip install -r requirements/production.txt
+
+# Copy over rest of code.
+# We do this AFTER requirements so that the requirements cache isn't busted
+# every time any bit of code is changed.
+COPY . .
+
+# Expose canonical Discovery port
+EXPOSE 8381
+
+CMD gunicorn --bind=0.0.0.0:8381 --workers 2 --max-requests=1000 -c course_discovery/docker_gunicorn_configuration.py course_discovery.wsgi:application
+
+FROM app as devstack
+ENV DISCOVERY_CFG /edx/app/discovery/devstack.yml
+RUN make static
+
+FROM app as newrelic
+RUN pip install newrelic
+CMD newrelic-admin run-program gunicorn --bind=0.0.0.0:8381 --workers 2 --max-requests=1000 -c course_discovery/docker_gunicorn_configuration.py course_discovery.wsgi:application