Merge pull request #14 from alan-turing-institute/python-migration

Migrate code to use Python throughout

.github/workflows/test_code.yaml

@@ -13,18 +13,32 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
-      - name: Install bats
-        run: sudo apt-get update && sudo apt-get install bats
-      - name: Install ruby
-        uses: ruby/setup-ruby@v1
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+
+      - name: Install hatch
+        run: pip install hatch
+
+      - name: Test Python
+        run: hatch run test:all
+
+      # For security reasons, PRs created from forks cannot generate PR comments directly
+      # (see https://securitylab.github.com/research/github-actions-preventing-pwn-requests/).
+      # Instead we need to trigger another workflow after this one completes.
+      - name: Generate coverage comment
+        id: coverage_comment
+        uses: py-cov-action/python-coverage-comment-action@v3
+        with:
+          GITHUB_TOKEN: ${{ github.token }}
+
+      # Save the coverage comment for later use
+      # See https://github.com/py-cov-action/python-coverage-comment-action/blob/main/README.md
+      - name: Save coverage comment as an artifact
+        uses: actions/upload-artifact@v4
+        if: steps.coverage_comment.outputs.COMMENT_FILE_WRITTEN == 'true'
         with:
-          ruby-version: 3.4
-      - name: Install ruby dependencies
-        run: gem install mustache
-      - name: Install yq
-        run: |
-          VERSION=v4.44.1
-          BINARY=yq_linux_amd64
-          wget https://github.com/mikefarah/yq/releases/download/${VERSION}/${BINARY}.tar.gz -O - | tar xz && sudo mv ${BINARY} /usr/bin/yq
-      - name: Run tests
-        run: bats tests
+          name: python-coverage-comment-action
+          path: python-coverage-comment-action.txt

.github/workflows/test_coverage.yaml

@@ -0,0 +1,32 @@
+---
+name: Test code - post coverage comment
+
+# Run workflow after test_code has completed
+on:  # yamllint disable-line rule:truthy
+  workflow_run:
+    workflows: ["Test code"]
+    types:
+      - completed
+
+jobs:
+  coverage:
+    runs-on: ubuntu-latest
+    if: github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success'
+    permissions:
+      # Gives the action the necessary permissions for publishing new
+      # comments in pull requests.
+      pull-requests: write
+      # Gives the action the necessary permissions for editing existing
+      # comments (to avoid publishing multiple comments in the same PR)
+      contents: write
+      # Gives the action the necessary permissions for looking up the
+      # workflow that launched this workflow, and download the related
+      # artifact that contains the comment to be published
+      actions: read
+    steps:
+      # Post the pre-generated coverage comment
+      - name: Post coverage comment
+        uses: py-cov-action/python-coverage-comment-action@v3
+        with:
+          GITHUB_TOKEN: ${{ github.token }}
+          GITHUB_PR_RUN_ID: ${{ github.event.workflow_run.id }}

.gitignore

@@ -0,0 +1,3 @@
+.coverage
+requirements.txt
+**/__pycache__/

Dockerfile

@@ -1,33 +1,91 @@
-FROM debian:stable-slim
+# Build working image
+FROM python:3.11.9-slim AS builder
 
-# Set up work directory
+## Set up work directory
 WORKDIR /app
 
-# Install prerequisites
-RUN apt-get update -y; \
-    apt upgrade -y; \
-    apt install -y \
-        cron \
+## Configure Python settings
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+
+## Install build prerequisites
+RUN apt-get update && \
+    apt-get install -y \
+        dumb-init \
+        g++ \
         gcc \
+        libldap2-dev \
         libpq-dev \
-        make \
-        postgresql-client \
-        ruby \
-        ruby-dev \
-        s6;
-
-# Install ruby requirements
-RUN gem install mustache pg-ldap-sync;
-
-# Copy required files
-COPY resources resources
-COPY scripts scripts
-COPY templates templates
-COPY synchronise /etc/s6/synchronise
-
-# Set file permissions
-RUN chmod 0700 /etc/s6/synchronise/* /app/scripts/*
-RUN chmod 0600 /app/resources/* /app/templates/*
-
-# Schedule jobs with s6
-CMD ["/bin/s6-svscan", "/etc/s6"]
+        libsasl2-dev \
+        patchelf \
+        pipx \
+        python3-dev \
+        wget \
+        && \
+    pipx install hatch
+
+## Copy project files needed by hatch
+COPY README.md pyproject.toml ./
+COPY guacamole_user_sync guacamole_user_sync
+
+## Build wheels for dependencies then use auditwheel to include shared libraries
+## Note that we need to specify psycopg[c] in order to ensure that dependencies are included in the wheel
+RUN /root/.local/bin/hatch run pip freeze | grep -v "^-e" > requirements.txt && \
+    sed -i "s/psycopg=/psycopg[c]=/g" requirements.txt && \
+    python -m pip wheel --no-cache-dir --no-binary :all: --wheel-dir /app/repairable -r requirements.txt && \
+    python -m pip install auditwheel && \
+    for WHEEL in /app/repairable/*.whl; do \
+        auditwheel repair --wheel-dir /app/wheels --plat manylinux_2_34_aarch64 "${WHEEL}" 2> /dev/null || mv "${WHEEL}" /app/wheels/; \
+    done;
+
+## Build a separate pip wheel which can be used to install itself
+RUN python -m pip wheel --no-cache-dir --wheel-dir /app/wheels pip && \
+    mv /app/wheels/pip*whl /app/wheels/pip-0-py3-none-any.whl
+
+## Build a separate wheel for the project
+RUN /root/.local/bin/hatch build -t wheel && \
+    mv dist/guacamole_user_sync*.whl /app/wheels/ && \
+    echo "guacamole-user-sync>=0.0" >> requirements.txt
+
+# Build final image
+FROM gcr.io/distroless/python3-debian12:debug
+
+## This shell is only available in the debug image
+SHELL ["/busybox/sh", "-c"]
+
+## Set up work directory
+WORKDIR /app
+
+## Configure Python settings
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+
+## Copy required files
+COPY --from=builder /app/wheels /tmp/wheels
+COPY --from=builder /app/requirements.txt .
+COPY --from=builder /usr/bin/dumb-init /usr/bin/dumb-init
+COPY synchronise.py .
+
+## Install pip from wheel
+RUN python /tmp/wheels/pip-0-py3-none-any.whl/pip install \
+    --break-system-packages \
+    --root-user-action ignore \
+    --no-index \
+    /tmp/wheels/pip-0-py3-none-any.whl && \
+    rm /tmp/wheels/pip-0-py3-none-any.whl
+
+## Install Python packages from wheels
+RUN python -m pip install \
+        --break-system-packages \
+        --root-user-action ignore \
+        --find-links /tmp/wheels/ \
+        -r /app/requirements.txt && \
+    rm -rf /tmp/wheels && \
+    python -m pip freeze
+
+## Set file permissions
+RUN chmod 0700 /app/synchronise.py
+
+## Run jobs with dumb-init
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]
+CMD ["python", "/app/synchronise.py"]

README.md

@@ -3,6 +3,7 @@ Synchronise a Guacamole PostgreSQL database with an LDAP server, such as Microso
 
 ## Environment variables
 
+- DEBUG: Enable debug output (default: 'False')
 - LDAP_BIND_DN: (Optional) distinguished name of LDAP bind user
 - LDAP_BIND_PASSWORD: (Optional) password of LDAP bind user
 - LDAP_GROUP_BASE_DN: Base DN for groups

guacamole_user_sync/__about__.py

@@ -0,0 +1 @@
+__version__ = "0.6.0"

guacamole_user_sync/__init__.py

@@ -0,0 +1,3 @@
+from .__about__ import __version__ as version
+
+__all__ = ["version"]

guacamole_user_sync/ldap/__init__.py

@@ -0,0 +1,5 @@
+from .ldap_client import LDAPClient
+
+__all__ = [
+    "LDAPClient",
+]

guacamole_user_sync/ldap/ldap_client.py

@@ -0,0 +1,103 @@
+import logging
+
+import ldap
+from ldap.asyncsearch import List as AsyncSearchList
+from ldap.ldapobject import LDAPObject
+
+from guacamole_user_sync.models import (
+    LDAPException,
+    LDAPGroup,
+    LDAPQuery,
+    LDAPSearchResult,
+    LDAPUser,
+)
+
+logger = logging.getLogger("guacamole_user_sync")
+
+
+class LDAPClient:
+    def __init__(
+        self,
+        hostname: str,
+        *,
+        bind_dn: str | None = None,
+        bind_password: str | None = None,
+    ) -> None:
+        self.cnxn: LDAPObject | None = None
+        self.bind_dn = bind_dn
+        self.bind_password = bind_password
+        self.hostname = hostname
+
+    def connect(self) -> LDAPObject:
+        if not self.cnxn:
+            logger.info(f"Initialising connection to LDAP host at {self.hostname}")
+            self.cnxn = ldap.initialize(f"ldap://{self.hostname}")
+            if self.bind_dn:
+                try:
+                    self.cnxn.simple_bind_s(self.bind_dn, self.bind_password)
+                except ldap.INVALID_CREDENTIALS as exc:
+                    logger.warning("Connection credentials were incorrect.")
+                    raise LDAPException from exc
+        return self.cnxn
+
+    def search_groups(self, query: LDAPQuery) -> list[LDAPGroup]:
+        output = []
+        for result in self.search(query):
+            attr_dict = result[1][1]
+            output.append(
+                LDAPGroup(
+                    member_of=[
+                        group.decode("utf-8") for group in attr_dict["memberOf"]
+                    ],
+                    member_uid=[
+                        group.decode("utf-8") for group in attr_dict["memberUid"]
+                    ],
+                    name=attr_dict[query.id_attr][0].decode("utf-8"),
+                )
+            )
+        logger.debug(f"Loaded {len(output)} LDAP groups")
+        return output
+
+    def search_users(self, query: LDAPQuery) -> list[LDAPUser]:
+        output = []
+        for result in self.search(query):
+            attr_dict = result[1][1]
+            output.append(
+                LDAPUser(
+                    display_name=attr_dict["displayName"][0].decode("utf-8"),
+                    member_of=[
+                        group.decode("utf-8") for group in attr_dict["memberOf"]
+                    ],
+                    name=attr_dict[query.id_attr][0].decode("utf-8"),
+                    uid=attr_dict["uid"][0].decode("utf-8"),
+                )
+            )
+        logger.debug(f"Loaded {len(output)} LDAP users")
+        return output
+
+    def search(self, query: LDAPQuery) -> LDAPSearchResult:
+        results: LDAPSearchResult = []
+        logger.info("Querying LDAP host with:")
+        logger.info(f"... base DN: {query.base_dn}")
+        logger.info(f"... filter: {query.filter}")
+        searcher = AsyncSearchList(self.connect())
+        try:
+            searcher.startSearch(
+                query.base_dn,
+                ldap.SCOPE_SUBTREE,
+                query.filter,
+            )
+            if searcher.processResults() != 0:
+                logger.warning("Only partial results received.")
+            results = searcher.allResults
+            logger.debug(f"Server returned {len(results)} results.")
+            return results
+        except ldap.NO_SUCH_OBJECT as exc:
+            logger.warning("Server returned no results.")
+            raise LDAPException from exc
+        except ldap.SERVER_DOWN as exc:
+            logger.warning("Server could not be reached.")
+            raise LDAPException from exc
+        except ldap.SIZELIMIT_EXCEEDED as exc:
+            logger.warning("Server-side size limit exceeded.")
+            raise LDAPException from exc

guacamole_user_sync/models/__init__.py

@@ -0,0 +1,14 @@
+from .exceptions import LDAPException, PostgreSQLException
+from .ldap_objects import LDAPGroup, LDAPUser
+from .ldap_query import LDAPQuery
+
+LDAPSearchResult = list[tuple[int, tuple[str, dict[str, list[bytes]]]]]
+
+__all__ = [
+    "LDAPException",
+    "LDAPGroup",
+    "LDAPQuery",
+    "LDAPSearchResult",
+    "LDAPUser",
+    "PostgreSQLException",
+]

guacamole_user_sync/models/exceptions.py

@@ -0,0 +1,6 @@
+class LDAPException(Exception):
+    pass
+
+
+class PostgreSQLException(Exception):
+    pass

guacamole_user_sync/models/ldap_objects.py

@@ -0,0 +1,20 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class LDAPGroup:
+    """An LDAP group with required attributes only."""
+
+    member_of: list[str]
+    member_uid: list[str]
+    name: str
+
+
+@dataclass
+class LDAPUser:
+    """An LDAP user with required attributes only."""
+
+    display_name: str
+    member_of: list[str]
+    name: str
+    uid: str

guacamole_user_sync/models/ldap_query.py

@@ -0,0 +1,10 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class LDAPQuery:
+    """An LDAP query with attributes."""
+
+    base_dn: str
+    filter: str
+    id_attr: str

guacamole_user_sync/postgresql/__init__.py

@@ -0,0 +1,9 @@
+from .postgresql_backend import PostgreSQLBackend
+from .postgresql_client import PostgreSQLClient
+from .sql import SchemaVersion
+
+__all__ = [
+    "PostgreSQLBackend",
+    "PostgreSQLClient",
+    "SchemaVersion",
+]