Merge pull request #129 from ReadAlongs/master

It's time for a release

.git-blame-ignore-revs

@@ -0,0 +1,5 @@
+02babfb698a8dfbeb87f5be6ad21172eba82bc05
+ee480fbd24c2d0b1730f5ae4a6be6c6bc842eb94
+1862060ef717c05080c9b47497dc79328563b072
+3416098be96c2e8efee5c5ce1e935711575d2e47
+13435428e87005f168db210019759bf7578ec06f

.github/workflows/pythonpublish.yml

@@ -9,21 +9,21 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Set up Python
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v4
         with:
           python-version: "3.x"
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install setuptools wheel twine
+          pip install build twine
       - name: Build and publish
         env:
           TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
           TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
         run: |
-          python setup.py sdist bdist_wheel
+          python -m build --sdist --wheel
           twine upload dist/*
       - name: Determine tag
         id: determine_tag

.github/workflows/tests.yml

@@ -0,0 +1,77 @@
+name: Run Tests
+on:
+  - push
+  - pull_request
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    # #no-ci in the commit log flags commit we don't want CI-validated
+    if: ${{ !contains(github.event.head_commit.message, '#no-ci') }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: FedericoCarboni/setup-ffmpeg@v2
+        id: setup-ffmpeg
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.7"
+          cache: "pip"
+
+      - name: Install Python dependencies
+        run: |
+          # Keep pip up to date
+          python -m pip install --upgrade pip
+          # Some dependencies are built using wheel
+          pip install wheel
+          # Install all Python dependencies in just one pip call, including Studio itself
+          pip install -r requirements.txt \
+                      -r requirements.dev.txt \
+                      -r requirements.ci.txt \
+                      -e .
+
+      - name: Run tests
+        run: |
+          gunicorn readalongs.app:app --bind 0.0.0.0:5000 --daemon
+          cd test && coverage run run.py prod && coverage xml
+
+      - name: Nitpicking
+        run: |
+          # coding style: we want black compliance
+          find . -name \*.py | xargs black --check
+          # Legal check: make sure we don't have or introduce GPL dependencies
+          if pip-licenses | grep -v 'Artistic License' | grep -v LGPL | grep GNU; then echo 'Please avoid introducing *GPL dependencies'; false; fi
+
+      - uses: codecov/codecov-action@v3
+        with:
+          directory: ./test
+          token: ${{ secrets.CODECOV_TOKEN }}  # optional but apparently makes upload more reliable
+          fail_ci_if_error: false # too many upload errors to keep "true"
+
+  test-on-windows:
+    runs-on: windows-latest
+    if: ${{ !contains(github.event.head_commit.message, '#no-ci') }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: FedericoCarboni/setup-ffmpeg@v2
+        id: setup-ffmpeg
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.7"
+          cache: "pip"
+
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install wheel
+          pip install -r requirements.txt `
+                      -r requirements.dev.txt `
+                      -r requirements.ci.txt `
+                      -e .
+
+      - name: Run tests on Windows
+        run: cd test && python run.py prod

.gitlint

@@ -0,0 +1,9 @@
+[general]
+# Enable conventional commit linting
+contrib=contrib-title-conventional-commits
+
+# Ignore any data sent to gitlint via stdin (helpful on Windows)
+ignore-stdin=true
+
+# We don't require a body, just a title, even though a body is also a good idea
+ignore=body-is-missing

.pre-commit-config.yaml

@@ -12,6 +12,8 @@ repos:
     -   id: flake8
 -   repo: local
     # Using local repos because these won't work for me from remote repo -EJ
+    # They're also more convenient because we install them via requirements.dev.txt
+    # and they are then available on the command line as well as in pre-commit.
     hooks:
     -   id: isort
         name: isort
@@ -25,7 +27,9 @@ repos:
         language: system
         types: [python]
         stages: [commit]
--   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 'v0.782'  # Use the sha / tag you want to point at
-    hooks:
     -   id: mypy
+        name: mypy
+        entry: mypy
+        language: system
+        types: [python]
+        stages: [commit]

.pylintrc

@@ -1,7 +1,14 @@
 [MASTER]
 # A lot of test cases depend on etree, let's allow pylint to load it
 extension-pkg-allow-list=lxml.etree
-# We use isort for sorting our imports, so nevermind what pylint thinks
-disable=wrong-import-order
+
+disable=
+    # We use isort for sorting our imports, so nevermind what pylint thinks
+    wrong-import-order,
+    # I find the "unnecessary" else makes code more readable
+    no-else-return,
+    # We use single letter e for exception, f for file handles
+    invalid-name
+
 # Add . to the PYTHONPATH so pylint knows test cases can import basic_test_case
 init-hook="import sys; sys.path.append('.')"

.readthedocs.yml

@@ -1,13 +1,18 @@
 version: 2
 
 build:
-    os: ubuntu-20.04
-    tools:
-        python: "3.7"
+  os: ubuntu-20.04
+  tools:
+    python: "3.7"
+  jobs:
+    post_install:
+      - echo "Installing Studio itself in its current state"
+      - which pip python
+      - pip install -e .
 
 sphinx:
-    configuration: docs/conf.py
+  configuration: docs/conf.py
 
 python:
-    install:
-        - requirements: docs/requirements.txt
+  install:
+    - requirements: docs/requirements.txt

CITATION.cff

@@ -0,0 +1,46 @@
+cff-version: 1.2.0
+message: >-
+  If you use this software in a project of yours and write about it, please
+  cite our SIGUL 2022 paper using the following citation data.
+title: ReadAlongs Studio
+url: https://github.com/ReadAlongs/Studio
+preferred-citation:
+  type: conference-paper
+  title: >-
+    ReadAlong Studio: Practical Zero-Shot Text-Speech Alignment for Indigenous
+    Language Audiobooks
+  authors:
+    - given-names: Patrick
+      family-names: Littell
+      email: [email protected]
+      affiliation: National Research Council Canada
+    - given-names: Eric
+      family-names: Joanis
+      email: [email protected]
+      affiliation: National Research Council Canada
+    - given-names: Aidan
+      family-names: Pine
+      email: [email protected]
+      affiliation: National Research Council Canada
+    - given-names: Marc
+      family-names: Tessier
+      email: [email protected]
+      affiliation: National Research Council Canada
+    - given-names: David
+      family-names: Huggins-Daines
+      email: [email protected]
+      affiliation: Independent Researcher
+    - given-names: Delasie
+      family-names: Torkornoo
+      email: [email protected]
+      affiliation: Carleton University
+  collection-title: Proceedings of SIGUL2022 @LREC2022
+  start: 23
+  end: 32
+  year: 2022
+  month: 6
+  publisher:
+    name: European Language Resources Assiciation (ELRA)
+  location:
+    name: Marseille
+  url: http://www.lrec-conf.org/proceedings/lrec2022/workshops/SIGUL/pdf/2022.sigul-1.4.pdf

Contributing.md

@@ -16,15 +16,16 @@ commits.
 Run these commands in each of your sandboxes to enable our pre-commit hooks and commitlint:
 
 ```sh
+pip install -r requirements.dev.txt
 pre-commit install
-npm install
+gitlint install-hook
 ```
 
 ## Pre-commit hooks
 
 The ReadAlong Studio team has agreed to systematically use a number of pre-commit hooks to
 normalize formatting of code. You need to install and enable pre-commit to have these used
-when you do your own commits.
+automatically when you do your own commits.
 
 Pre-commit hooks enabled:
 - check-yaml validates YAML files
@@ -60,11 +61,11 @@ don't forget to do so when you clone a new sandbox!
 
 ## commitlint
 
-The team has also agreed to use commitlint-style commit messages. Install and enable
-[commitlint](https://github.com/conventional-changelog/commitlint) to have your commits
-validated systematically.
+The team has also agreed to use [Conventional Commits](https://www.conventionalcommits.org/).
+Install and enable [gitlint](https://jorisroovers.com/gitlint/) to have your
+commit messages scanned automatically.
 
-Commitlint commits look like this:
+Convential commits look like this:
 
     type(optional-scope): subject (i.e., short description)
 
@@ -107,32 +108,14 @@ These rules are inspired by these commit formatting guides:
 
 ### Enabling commitlint
 
-We run commitlint on each commit message that you write by enabling the commit-msg hook in
-Git. It is run via [husky](https://www.npmjs.com/package/husky), which is a JS Git hook
-manager, and you need Node to run it.
-
-If you don't already use Node, this is a bit more work to install that the pre-commit
-hooks above, but please take a moment to do this:
+You can run commitlint on each commit message that you write by enabling the
+commit-msg hook in Git.
 
-- If you don't already use Node or nvm, or if you don't have admin access to the system
-  version of node, install nvm in your ~/.nvm folder:
-```sh
-wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash
-```
-This will add a few lines to your `.bashrc` file, which you'll need to execute now,
-possibly by starting a new shell.
-
-- Install Node:
-```sh
-nvm install node
-```
+Run this command in your g2p sandbox to install and enable the commit-msg hook:
 
-- In your ReadAlong/Studio sandbox, install the husky commit-msg hook using npm, the node
-  package manager you just installed using nvm. The file `package.json` in Studio is what
-  tells npm to install husky as a pre-commit hook, and also what tells husky to invoke
-  commitlint on your commit messages.
 ```sh
-npm install
+pip install -r requirements/requirements.dev.txt
+gitlint install-hook
 ```
 
 - Now, next time you make a change and commit it, your commit log will be checked:

Dockerfile

@@ -3,40 +3,46 @@ FROM debian:bullseye-slim
 ENV APPHOME /opt/readalong-studio
 ENV PORT 5000
 
-# Install system dependencies
-#  - swig: required by pocketsphinx
-#  - libpulse-dev: required by pocketsphinx
-#  - portaudio19-dev: required by pocketsphinx
-RUN apt-get update && apt-get install -y \
+# Lean, optimized installation of system dependencies
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends --yes \
         python3 \
         python3-pip \
         git \
-        swig \
-        libpulse-dev \
-        portaudio19-dev \
         ffmpeg \
-        vim-nox
+        vim-nox \
+	less \
+    && apt-get clean \
+    && apt-get autoremove \
+    && rm -fr /var/lib/apt/lists/*
 
 # Install 3rd party dependencies in their own layer, for faster rebuilds when we
 # change ReadAlong-Studio source code
-RUN python3 -m pip install gevent
-ADD requirements.txt $APPHOME/requirements.txt
-RUN python3 -m pip install -r $APPHOME/requirements.txt
-# RUN python3 -m pip install gunicorn # If you want to run production server
+ADD requirements.* $APPHOME/
+RUN python3 -m pip install --upgrade pip \
+    && python3 -m pip install -r $APPHOME/requirements.txt \
+    && python3 -m pip install gevent
 
 # We don't want Docker to cache the installation of g2p or Studio, so place them
 # after COPY . $APPHOME, which almost invariable invalidates the cache.
 COPY . $APPHOME
 WORKDIR $APPHOME
 # Get and install the latest g2p
-RUN git clone https://github.com/roedoejet/g2p.git
-RUN cd g2p && python3 -m pip install -e .
+RUN git clone https://github.com/roedoejet/g2p.git \
+    && cd g2p \
+    && python3 -m pip install -e .
+
 # Install ReadAlong-Studio itself
 RUN python3 -m pip install -e .
 
-# Run the default gui (on localhost:5000)
+# Run the default gui (on localhost:5000, make sure you use -p 5000:5000 when
+# you docker run the image)
 CMD python3 ./run.py
 
 # For a production server, comment out the default gui CMD above, and run the
 # gui using gunicorn instead:
-# CMD gunicorn -k gevent -w 1 readalongs.app:app --bind 0.0.0.0:5000
+# CMD gunicorn -k gevent -w 1 readalongs.app:app --bind 0.0.0.0:$PORT
+
+# For the web API, use this CMD instead, the same on our Heroku deployment, except
+# with binding to port 5000
+# CMD gunicorn -w 4 -k uvicorn.workers.UvicornWorker readalongs.web_api:web_api_app --bind 0.0.0.0:$PORT

LICENSE

@@ -2,6 +2,7 @@ MIT License
 
 Copyright (c) 2019 David Huggins-Daines
 Copyright (c) 2019-2021 National Research Council Canada
+Acoustic model in readalongs/static/model/cmusphinx-en-us-5.2 Copyright (c) 2015 Alpha Cephei Inc. licensed under the FreeBSD License; see README in that directory.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Procfile

@@ -0,0 +1,2 @@
+# Command for launching the web API server for ReadAlongs-Studio on Heroku
+web: gunicorn -w 4 -k uvicorn.workers.UvicornWorker readalongs.web_api:web_api_app