Merge pull request #53 from lsst-sqre/tickets/DM-40187

DM-40187: Add an "accept list" of GitHub organizations

.gitignore

@@ -1,5 +1,3 @@
-docs/_static/openapi.json
-
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -72,6 +70,7 @@ instance/
 
 # Sphinx documentation
 docs/_build/
+docs/_static/openapi.json
 
 # PyBuilder
 target/

CHANGELOG.md

@@ -8,6 +8,24 @@ Collect fragments into this file with: scriv collect --version X.Y.Z
 
 <!-- scriv-insert-here -->
 
+<a id='changelog-0.8.0'></a>
+## 0.8.0 (2023-07-27)
+
+### New features
+
+- Add a new `TS_GITHUB_ORGS` environment variable. This can be set to a comma-separated list of GitHub organizations that can install the Times Square GitHub App and sync notebooks into the Times Square service. This is a an important security feature if the Times Square GitHub App is set to public so that multiple GitHub organizations can sync repositories with Times Square. GitHub webhook handlers filter out events from non-accepted organizations. The `GitHubRepoService` also checks the ownership on initialization.
+
+### Other changes
+
+- Adopt scriv for managing the changelog.
+- Adopt ruff for linting, and update the codebase accordingly.
+- Adopt the new neophile workflow for managing dependencies.
+- Adopt the new `lsst-sqre/build-and-push-to-ghcr` GitHub Action for building and pushing Docker images.
+- Adopt the new FastAPI lifespan function for handling start-up and shutdown.
+- Create a Sphinx documentation site at `times-square.lsst.io`.
+
+- Add documentation for configuring the Times Square GitHub App, including a sample URL with the app settings built-in.
+
 ## 0.7.0 (2023-04-19)
 
 ### New features

docs/conf.py

@@ -1 +1,98 @@
+from pathlib import Path
+from urllib.parse import quote, urlencode
+
 from documenteer.conf.guide import *
+
+
+def create_github_app_qs(domain="data-dev.lsst.cloud"):
+    """Create a URL query string with the GitHub App configuration.
+
+    See
+    https://docs.github.com/en/apps/creating-github-apps/setting-up-a-github-app/creating-a-github-app-using-url-parameters
+
+    Parameters
+    ----------
+    domain : `str`, optional
+        The domain name of the RSP environment. This is used to template some
+        of the GitHub App configuration.
+
+    Returns
+    -------
+    qs : `str`
+        The query string with the GitHub App configuration.
+    """
+    parameters = [
+        ("name", f"Times Square ({domain})"),
+        (
+            "description",
+            "Times Square is a service for parameterized Jupyter Notebooks as "
+            "dynamic webpages. An instance of the Times Square app "
+            "is associated with each "
+            "[RSP environment]("
+            "https://phalanx.lsst.io/environments/index.html).",
+        ),
+        ("url", "https://{domain}/times-square/"),
+        ("public", "false"),
+        ("webhook_active", "true"),
+        ("webhook_url", f"https://{domain}/times-square/api/github/webhook"),
+        ("events[]", "push"),
+        ("events[]", "check_run"),
+        ("events[]", "check_suite"),
+        ("events[]", "pull_request"),
+        ("events[]", "repository"),
+        ("contents", "read"),
+        ("pull_requests", "read"),
+        ("checks", "write"),
+    ]
+    return urlencode(parameters, quote_via=quote)
+
+
+def format_org_url(org="lsst-sqre", domain="data-dev.lsst.cloud"):
+    """Format the URL creating the app for an organization.
+
+    Parameters
+    ----------
+    org : `str`, optional
+        The GitHub organization name where the app is created.
+    domain : `str`, optional
+        The domain name of the RSP environment. This is used to template some
+        of the GitHub App configuration.
+
+    Returns
+    -------
+    url : `str`
+        The URL to create the GitHub App.
+    """
+    qs = create_github_app_qs(domain=domain)
+    url = f"https://github.com/organizations/{org}/settings/apps/new?{qs}"
+    return url
+
+
+def format_personal_url(domain="data-dev.lsst.cloud"):
+    """Format the URL creating the app for a user account.
+
+    Parameters
+    ----------
+    domain : `str`, optional
+        The domain name of the RSP environment. This is used to template some
+        of the GitHub App configuration.
+
+    Returns
+    -------
+    url : `str`
+        The URL to create the GitHub App.
+    """
+    qs = create_github_app_qs(domain=domain)
+    url = f"https://github.com/settings/apps/new?{qs}"
+    return url
+
+
+# Generate the URLs and write them as files that are included as code blocks
+# in the Sphinx documentation.
+org_install_path = Path("user-guide/_github-app-url-org.txt")
+org_install_path.write_text(format_org_url())
+
+personal_install_path = Path("user-guide/_github-app-url-personal.txt")
+personal_install_path.write_text(format_personal_url())
+
+exclude_patterns.append("**/_*.txt")

docs/user-guide/_github-app-url-org.txt

@@ -0,0 +1 @@
+https://github.com/organizations/lsst-sqre/settings/apps/new?name=Times%20Square%20%28data-dev.lsst.cloud%29&description=Times%20Square%20is%20a%20service%20for%20parameterized%20Jupyter%20Notebooks%20as%20dynamic%20webpages.%20An%20instance%20of%20the%20Times%20Square%20app%20is%20associated%20with%20each%20%5BRSP%20environment%5D%28https%3A%2F%2Fphalanx.lsst.io%2Fenvironments%2Findex.html%29.&url=https%3A%2F%2F%7Bdomain%7D%2Ftimes-square%2F&public=false&webhook_active=true&webhook_url=https%3A%2F%2Fdata-dev.lsst.cloud%2Ftimes-square%2Fapi%2Fgithub%2Fwebhook&events%5B%5D=push&events%5B%5D=check_run&events%5B%5D=check_suite&events%5B%5D=pull_request&events%5B%5D=repository&contents=read&pull_requests=read&checks=write

docs/user-guide/_github-app-url-personal.txt

@@ -0,0 +1 @@
+https://github.com/settings/apps/new?name=Times%20Square%20%28data-dev.lsst.cloud%29&description=Times%20Square%20is%20a%20service%20for%20parameterized%20Jupyter%20Notebooks%20as%20dynamic%20webpages.%20An%20instance%20of%20the%20Times%20Square%20app%20is%20associated%20with%20each%20%5BRSP%20environment%5D%28https%3A%2F%2Fphalanx.lsst.io%2Fenvironments%2Findex.html%29.&url=https%3A%2F%2F%7Bdomain%7D%2Ftimes-square%2F&public=false&webhook_active=true&webhook_url=https%3A%2F%2Fdata-dev.lsst.cloud%2Ftimes-square%2Fapi%2Fgithub%2Fwebhook&events%5B%5D=push&events%5B%5D=check_run&events%5B%5D=check_suite&events%5B%5D=pull_request&events%5B%5D=repository&contents=read&pull_requests=read&checks=write

docs/user-guide/environment-variables.rst

@@ -70,3 +70,7 @@ See the `Phalanx documentation for Times Square <https://phalanx.lsst.io/applica
 .. envvar:: TS_ENABLE_GITHUB_APP
 
    (boolean, default: true) Enable the GitHub App integration.
+
+.. envvar:: TS_GITHUB_ORGS
+
+   (string) A comma-separated list of GitHub organizations that Times Square will sync notebooks from. This is used to filter out GitHub App installations from the general public.

docs/user-guide/github-app-config.rst

@@ -0,0 +1,123 @@
+##########################
+Configuring the GitHub App
+##########################
+
+Times Square operates as a GitHub App to access notebooks in GitHub repositories, to get webhooks when those notebooks change, and to offer status checks for pull requests that change notebooks.
+Each installation of Times Square in different Phalanx environments has its own GitHub App in order to receive webhook events.
+Times Square installations in different RSP/Phalanx environments can share the same source repository, however.
+
+To learn more about installing GitHub Apps in general, see the `GitHub Apps documentation <https://docs.github.com/en/apps/creating-github-apps/setting-up-a-github-app/creating-a-github-app>`__.
+
+Create an app with a template URL
+=================================
+
+You can create the GitHub App by customizing and visiting the following URL (replace ``lsst-sqre`` with the GitHub organization or user that owns the source repository):
+
+.. literalinclude:: _github-app-url-org.txt
+
+Alternatively, the app can be installed in a personal account (not recommended for production use):
+
+.. literalinclude:: _github-app-url-personal.txt
+
+Once you follow the link, you will be able to make further customizations to the GitHub App before creating it.
+These settings are described in the following sections.
+
+GitHub App settings
+===================
+
+Name
+----
+
+The name of the GitHub App should be "Times Square (env)".
+For example, ``Times Square (data.lsst.cloud)``.
+
+This naming convention distinguishes the Times Square installations for each Phalanx environment.
+
+Description
+-----------
+
+Use the description provided with the GitHub App template URL, and modify it as needed.
+
+Homepage URL
+------------
+
+Set this to the Times Square app in the RSP, e.g. https://data-dev.lsst.cloud/times-square/.
+
+Identifying and authorizing users
+---------------------------------
+
+Not applicable.
+
+Post installation
+-----------------
+
+Not applicable.
+
+Webhook
+-------
+
+The webhook should be enabled.
+Set the webhook URL to the ``/times-square/api/github/webhook`` endpoint in the RSP/Phalanx environment.
+For example, ``https://data.lsst.cloud/times-square/api/github/webhook``.
+
+Create a webhook secret and store it in the :envvar:`TS_GITHUB_WEBHOOK_SECRET` environment variable (through Vault/1Password).
+
+Permissions
+-----------
+
+The GitHub App needs the following repository permissions:
+
+- **Checks**: Read & write
+- **Contents**: Read-only
+- **Metadata**: Read-only
+- **Pull requests**: Read-only
+
+Events
+------
+
+The GitHub App needs to subscribe to the following events:
+
+- Check Run
+- Check Suite
+- Push
+- Pull request
+- Repository
+
+.. _github-app-secrets:
+
+Create the app and secrets
+==========================
+
+Once the GitHub App is configured, you can click the :guilabel:`Create GitHub App` button to create it in your GitHub organization or user account.
+
+When you do this, you can create the secret keys that Times Square needs to authenticate with GitHub and verify webhook events.
+These are provided to Times Square as environment variables:
+
+- :envvar:`TS_GITHUB_APP_ID`: The GitHub App ID. This is shown on the GitHub App's :guilabel:`General` page under the :guilabel:`About` heading.
+- :envvar:`TS_GITHUB_APP_PRIVATE_KEY`: The GitHub App's private key. This is shown on the GitHub App's :guilabel:`General` page under :guilabel:`Client secrets`.
+- :envvar:`TS_GITHUB_WEBHOOK_SECRET`: The webhook secret you created in the GitHub App's :guilabel:`General` page under :guilabel:`Webhooks`.
+- :envvar:`TS_GITHUB_ORGS`: A comma-separated list of the GitHub organizations that Times Square should operate on. For a private GitHub App, this should be the organization that owns the app. See also: :ref:`multiple-github-orgs`.
+
+See :doc:`environment-variables` for more information on Phalanx's environment variable settings.
+
+Install the app in the source repository
+========================================
+
+Once the app is created and the Times Square app is configured, you need to *install* the app in the source repository (or repositories, if there are several).
+From the app's GitHub settings page, click :guilabel:`Install App` and select the repositories to install it in.
+Avoid installing the app in repositories that do not use Times Square.
+
+.. _multiple-github-orgs:
+
+Sourcing notebooks from multiple GitHub organizations
+=====================================================
+
+By default, the Times Square GitHub App is configured as "private," meaning that it can only be installed in repositories owned by the organization that owns the GitHub App.
+If you want to source notebooks from multiple GitHub organizations, you need to configure the GitHub App as "public" instead.
+There are three steps involved in doing this:
+
+1. Update the :envvar:`TS_GITHUB_ORGS` environment variable in the Times Square configuration to include the additional organizations. For example, set it to ``lsst,lsst-dm,lsst-sqre``.
+
+2. Update the GitHub App to be "public" instead of "private." This is done on the GitHub App's :guilabel:`Advanced` settings page under the :guilabel:`Make this GitHub App public` heading.
+
+3. Install the GitHub App in additional repositories in the other GitHub organizations.

docs/user-guide/index.rst

@@ -6,4 +6,5 @@ User guide
    :maxdepth: 2
    :caption: Deployment
 
+   github-app-config
    environment-variables

requirements/dev.in

@@ -15,6 +15,7 @@ pre-commit
 pytest
 pytest-asyncio
 pytest-cov
+pytest-mock
 uvicorn
 sqlalchemy[mypy]
 holdup

requirements/dev.txt

@@ -185,7 +185,7 @@ pycparser==2.21
     # via
     #   -c requirements/main.txt
     #   cffi
-pydantic==1.10.11
+pydantic==1.10.12
     # via
     #   -c requirements/main.txt
     #   documenteer
@@ -204,10 +204,13 @@ pytest==7.4.0
     #   -r requirements/dev.in
     #   pytest-asyncio
     #   pytest-cov
+    #   pytest-mock
 pytest-asyncio==0.21.1
     # via -r requirements/dev.in
 pytest-cov==4.1.0
     # via -r requirements/dev.in
+pytest-mock==3.11.1
+    # via -r requirements/dev.in
 python-dateutil==2.8.2
     # via
     #   -c requirements/main.txt

requirements/main.txt

@@ -38,7 +38,7 @@ cryptography==41.0.2
     #   safir
 defusedxml==0.7.1
     # via nbconvert
-dnspython==2.4.0
+dnspython==2.4.1
     # via email-validator
 email-validator==2.0.0.post2
     # via pydantic
@@ -63,9 +63,7 @@ h11==0.14.0
 hiredis==2.2.3
     # via redis
 httpcore==0.17.3
-    # via
-    #   dnspython
-    #   httpx
+    # via httpx
 httptools==0.6.0
     # via uvicorn
 httpx==0.24.1
@@ -126,7 +124,7 @@ mistune==3.0.1
     # via nbconvert
 nbclient==0.8.0
     # via nbconvert
-nbconvert==7.7.2
+nbconvert==7.7.3
     # via -r requirements/main.in
 nbformat==5.9.1
     # via
@@ -143,7 +141,7 @@ platformdirs==3.9.1
     # via jupyter-core
 pycparser==2.21
     # via cffi
-pydantic[email]==1.10.11
+pydantic[email]==1.10.12
     # via
     #   -r requirements/main.in
     #   fastapi
@@ -183,7 +181,6 @@ six==1.16.0
 sniffio==1.3.0
     # via
     #   anyio
-    #   dnspython
     #   httpcore
     #   httpx
 soupsieve==2.4.1

src/timessquare/config.py

@@ -139,6 +139,15 @@ class Config(BaseSettings):
         ),
     )
 
+    accepted_github_orgs: list[str] = Field(
+        env="TS_GITHUB_ORGS",
+        description=(
+            "A comma-separated list of GitHub organizations that can sync"
+            "with Times Square."
+        ),
+        default_factory=lambda: ["lsst-sqre"],
+    )
+
     redis_queue_url: RedisDsn = Field(
         env="TS_REDIS_QUEUE_URL",
         default_factory=lambda: RedisDsn(
@@ -161,6 +170,13 @@ class Config(BaseSettings):
         ),
     )
 
+    class Config:
+        @classmethod
+        def parse_env_var(cls, field_name: str, raw_val: str) -> Any:
+            if field_name == "accepted_github_orgs":
+                return [v.strip() for v in raw_val.split(",")]
+            return cls.json_loads(raw_val)  # type: ignore [attr-defined]
+
     @validator("path_prefix")
     def validate_path_prefix(cls, v: str) -> str:
         # Handle empty path prefix (i.e. app is hosted on its own domain)