DO NOT MERGE: Repository consolidation testing

.github/dependabot.yml

@@ -0,0 +1,11 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "weekly"

.github/profiles.yml

@@ -0,0 +1,16 @@
+# This is a dbt profile for CI use.
+# https://docs.getdbt.com/docs/core/connect-data-platform/connection-profiles
+aspects: # this needs to match the profile in your dbt_project.yml file
+  target: dev
+  outputs:
+    dev:
+      type: clickhouse
+      schema: xapi
+      host: localhost
+      port: 8123
+      user: ch_admin
+      password: "ch_password"
+      secure: False
+
+      custom_settings:
+        check_table_dependencies: 0

.github/workflows/ci-dbt-coverage.yml

@@ -0,0 +1,60 @@
+# Check documentation coverage
+
+name: dbt Tests & Coverage
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+env:
+  DBT_PROFILES_DIR: ./.github/
+  CLICKHOUSE_DB: "xapi"
+  CLICKHOUSE_USER: "ch_admin"
+  CLICKHOUSE_PASSWORD: "ch_password"
+  TUTOR_ROOT: ./dbt/.ci/
+
+jobs:
+  build:
+    name: Check coverage & run tests
+    runs-on: ubuntu-latest
+    permissions:
+      contents: "read"
+      id-token: "write"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12.x"
+      - name: Install dependencies
+        run: |
+          pip install tutor-contrib-aspects
+          pip install -r requirements/dbt.txt
+          cd dbt
+          pwd
+          ls -la
+          dbt deps
+      - name: Initialize Open edX
+        continue-on-error: false
+        run: |
+          tutor plugins enable aspects
+          tutor config save
+          tutor local start ralph -d
+          tutor local do init-clickhouse
+          tutor local do alembic -c "upgrade head"
+          tutor local do dbt
+          tutor local do load-xapi-test-data
+      - name: Check dbt tests
+        run: |
+          mv unit-test-seeds ci-seeds
+          dbt seed --full-refresh --selector all_tests
+          dbt run --full-refresh --selector all_tests
+          dbt test --selector all_tests
+          mv ci-seeds unit-test-seeds
+      - name: Check docs coverage
+        run: |
+          dbt docs generate
+          dbt-coverage compute doc --cov-fail-under 1.0 --model-path-filter models/

.github/workflows/ci-dbt-documentation.yml

@@ -0,0 +1,58 @@
+# Build documentation for dbt-aspects on every PR to main.
+# Deploy documentation to gh-pages branch on every push to main.
+
+name: Build documentation
+
+on:
+  push:
+    branches:
+      - main
+
+env:
+  DBT_PROFILES_DIR: ./.github/
+  CLICKHOUSE_DB: "xapi"
+  CLICKHOUSE_USER: "ch_admin"
+  CLICKHOUSE_PASSWORD: "ch_password"
+  TUTOR_ROOT: ./.ci/
+
+jobs:
+  build:
+    name: Deploy dbt docs to github pages
+    runs-on: ubuntu-latest
+    permissions:
+      contents: "read"
+      id-token: "write"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.SEMANTIC_RELEASE_GITHUB_TOKEN }}
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11.x"
+      - name: Install dependencies
+        run: |
+          pip install tutor-contrib-aspects
+          pip install -r requirements/dbt.txt
+          cd dbt
+          dbt deps
+      - name: Initialize Open edX
+        continue-on-error: true
+        run: |
+          tutor plugins enable aspects
+          tutor config save
+          tutor local start -d
+          tutor local do init -l aspects
+          tutor local do load-xapi-test-data
+      - name: Build docs
+        run: |
+          dbt run
+          dbt docs generate
+      - name: Deploy
+        if: github.event_name == 'push'
+        uses: peaceiris/actions-gh-pages@v4
+        with:
+          github_token: ${{ secrets.SEMANTIC_RELEASE_GITHUB_TOKEN }}
+          publish_dir: ./target
+          commit_message: "docs: update docs for "

.github/workflows/ci-dbt-format.yml

@@ -0,0 +1,22 @@
+name: Validate SQL format
+
+on:
+  - pull_request
+
+jobs:
+  format:
+    name: Validate SQL format
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11.x"
+      - name: Install dependencies
+        run: |
+          pip install -r requirements/dbt.txt
+      - name: Format SQL
+        run: |
+          sqlfmt dbt/models dbt/macros --check

.github/workflows/ci-python.yml

@@ -0,0 +1,39 @@
+name: Run all tests & checks
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  run_tests:
+    name: tests
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [ '3.11','3.12' ]
+    permissions:
+      # Gives the action the necessary permissions for publishing new
+      # comments in pull requests.
+      pull-requests: write
+      # Gives the action the necessary permissions for pushing data to the
+      # python-coverage-comment-action branch, and for editing existing
+      # comments (to avoid publishing multiple comments in the same PR)
+      contents: write
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: setup python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install pip
+        run: pip install -r requirements/pip.txt
+
+      - name: Install Dependencies
+        run: pip install -r requirements/ci.txt
+
+      - name: Run Tests
+        run: tox

.github/workflows/upgrade-python-requirements.yml

@@ -0,0 +1,30 @@
+name: Upgrade Python Requirements
+
+on:
+  schedule:
+    - cron: "0 0 * * 1"
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: "Target branch against which to create requirements PR"
+        required: true
+        # If copying this template manually, you must provide your default branch name
+        # in quotes, such as 'master'
+        default: 'main'
+
+jobs:
+  call-upgrade-python-requirements-workflow:
+    uses: openedx/.github/.github/workflows/upgrade-python-requirements.yml@master
+    with:
+    # If copying manually, also provide your default branch name in quotes here
+      branch: ${{ github.event.inputs.branch || 'main' }}
+      # optional parameters below; fill in if you'd like github or email notifications
+      # user_reviewers: ""
+      team_reviewers: 'axim-aximprovements'
+      email_address: '[email protected]'
+      # send_success_notification: false
+    secrets:
+      requirements_bot_github_token: ${{ secrets.REQUIREMENTS_BOT_GITHUB_TOKEN }}
+      requirements_bot_github_email: ${{ secrets.REQUIREMENTS_BOT_GITHUB_EMAIL }}
+      edx_smtp_username: ${{ secrets.EDX_SMTP_USERNAME }}
+      edx_smtp_password: ${{ secrets.EDX_SMTP_PASSWORD }}

.gitignore

@@ -1,6 +1,14 @@
-build
-docs/_build
+__pycache__/
 .DS_Store
-source/.DS_Store
+.coverage
 .idea/
-.python-version
+.python-version
+.tox/
+.venv
+*.pyc
+*.egg-info/
+**/tests/__init__.py
+build
+coverage.xml
+dist/
+docs/_build

.readthedocs.yaml

@@ -14,11 +14,11 @@ sphinx:
 build:
   os: "ubuntu-22.04"
   tools:
-    python: "3.8"
+    python: "3.12"
   apt_packages:
     - graphviz
 
 # Optionally set the version of Python and requirements required to build your docs
 python:
   install:
-  - requirements: requirements/docs.txt
+    - requirements: requirements/docs.txt

MANIFEST.in

@@ -0,0 +1,6 @@
+include LICENSE
+include README.rst
+include requirements/base.in
+include requirements/constraints.txt
+recursive-include aspects *.html *.png *.gif *.js *.css *.jpg *.jpeg *.svg *.csv
+recursive-include dbt *

Makefile

@@ -13,7 +13,7 @@ help:
 	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 	@awk -F ':.*?## ' '/^[a-zA-Z]/ && NF==2 {printf "\033[36m  %-25s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) | sort
 
-.PHONY: help Makefile upgrade requirements
+.PHONY: help clean Makefile upgrade requirements dev-requirements quality selfcheck test test-all validate
 
 # Define PIP_COMPILE_OPTS=-v to get more information during make upgrade.
 PIP_COMPILE = pip-compile --upgrade $(PIP_COMPILE_OPTS)
@@ -27,18 +27,67 @@ upgrade: ## update the requirements/*.txt files with the latest packages satisfy
 	pip install -r requirements/pip-tools.txt
 	$(PIP_COMPILE) -o requirements/base.txt requirements/base.in
 	$(PIP_COMPILE) -o requirements/docs.txt requirements/docs.in
+	$(PIP_COMPILE) -o requirements/test.txt requirements/test.in
+	$(PIP_COMPILE) -o requirements/quality.txt requirements/quality.in
+	$(PIP_COMPILE) -o requirements/ci.txt requirements/ci.in
+	$(PIP_COMPILE) -o requirements/dev.txt requirements/dev.in
+	$(PIP_COMPILE) -o requirements/dbt.txt requirements/dbt.in
 
 requirements:
 	pip install -r requirements/pip-tools.txt
 	pip-sync requirements/base.txt
 
+dev-requirements:
+	pip install -r requirements/pip-tools.txt
+	pip-sync requirements/base.txt requirements/dev.txt requirements/dbt.txt
+
+dbt-requirements:
+	pip install -r requirements/pip-tools.txt
+	pip-sync requirements/dbt.txt
+
+quality: ## check coding style with pycodestyle and pylint
+	pylint aspects/ *.py
+	pycodestyle aspects  *.py
+	pydocstyle aspects *.py
+	isort --check-only --diff --recursive aspects *.py test_settings.py
+	python setup.py bdist_wheel
+	twine check dist/*
+	make selfcheck
+
 serve_docs:
 	sphinx-autobuild -W docs/ docs/_build/html/
 
 # Emulate the build step on RTD to flush out errors ahead pushing
 check_docs:
 	sphinx-build -T -E -W --keep-going docs/ docs/_build/html
 
+clean: ## remove generated byte code, coverage reports, and build artifacts
+	find . -name '__pycache__' -exec rm -rf {} +
+	find . -name '*.pyc' -exec rm -f {} +
+	find . -name '*.pyo' -exec rm -f {} +
+	find . -name '*~' -exec rm -f {} +
+	coverage erase
+	rm -fr build/
+	rm -fr dist/
+	rm -fr *.egg-info
+
+test: clean ## run tests in the current virtualenv
+	pytest
+
+test-all: quality clean ## run tests on every supported Python/Django combination
+	tox
+
+validate: quality test ## run tests and quality checks
+
+dbt-format:
+	sqlfmt dbt/models dbt/macros
+
+dbt-coverage: ## Must be run after a dbt compile or run as it needs manifest.json
+	dbt-coverage compute doc --cov-fail-under 1.0
+
+selfcheck: ## check that the Makefile is well-formed
+	@echo "The Makefile is well-formed."
+
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile

README.md → README.rst

@@ -6,9 +6,9 @@ Learner analytics for Open edX!
 
 Aspects is an optional implementation of analytics for the Open edX LMS. It is the combined solution of Cairn by Overhang.io and the OARS project developed by Axim Collaborative with a huge amount of help from the Open edX community. Primarily it is intended to be a "batteries included" set of configurations and plugins to combine 3rd party tools into a powerful and flexible system for learner analytics.
 
-## What _isn't_ Aspects?
+## This repository
 
-A deployable application in-and-of itself, it helps deploy other applications together using a Tutor plugin and customizations to the Open edX platform to connect well-supported, existing, third party applications.
+This repository holds the core Aspects codebase. These include things like the schema management tools for the data warehouse, the Superset assets and configuration scripts, and tools for helping test and deploy Aspects. It also includes the documentation for Aspects that rendered to the [Open edX documentation site](https://docs.openedx.org/projects/openedx-aspects).
 
 ## Status
 

aspects/__init__.py

@@ -0,0 +1,5 @@
+"""
+Python code for the Aspects project.
+"""
+
+__version__ = "2.0"

aspects/xapi_db_load/.gitignore

@@ -0,0 +1,2 @@
+private_configs
+logs