Initializing gh-pages branch

.github/workflows/main.yml

@@ -0,0 +1,18 @@
+name: Run tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  test:
+    name: Run tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - name: Build containers and run tests
+        run: docker-compose -f docker-compose.yml -f tests/docker-compose.yml run --rm scrapers

.github/workflows/publish_image.yml

@@ -0,0 +1,29 @@
+name: Publish Docker image to GitHub Container Registry
+
+on:
+  push:
+    branches:
+      - main
+      - deploy
+
+jobs:
+  publish:
+    name: Publish Docker image to GitHub Container Registry
+    runs-on: ubuntu-latest
+    steps:
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Create tag from branch
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: ghcr.io/${{ github.repository }}
+      - name: Push to GitHub Container Registry
+        uses: docker/build-push-action@v2
+        with:
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}

.gitignore

@@ -0,0 +1,63 @@
+.#*
+*#
+
+# Pupa bits.
+_data
+_cache
+
+# scraper bits
+HearingSchedule.pdf
+HearingSchedule.txt
+
+# vim
+*swp
+
+# Python objects
+*.py[cod]
+
+# C extensions
+*.so
+
+# Packages
+*.egg
+*.egg-info
+dist
+build
+eggs
+parts
+bin
+var
+sdist
+develop-eggs
+.installed.cfg
+lib
+lib64
+
+# Installer logs
+pip-log.txt
+
+# Unit test / coverage reports
+.coverage
+.tox
+nosetests.xml
+
+# Translations
+*.mo
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+
+__pycache__
+
+venv
+
+/keyrings/live/pubring.gpg~
+/keyrings/live/pubring.kbx~
+/keyrings/live/secring.gpg
+
+# Encrypted files
+lametro/secrets.py
+
+.DS_Store

Dockerfile

@@ -0,0 +1,33 @@
+FROM ubuntu:20.04 as builder
+
+# Clone and build Blackbox
+RUN apt-get update && apt-get install -y build-essential git-core && \
+    git clone https://github.com/StackExchange/blackbox.git && \
+    cd blackbox && \
+    make copy-install
+
+FROM python:3.10
+LABEL maintainer "DataMade <[email protected]>"
+
+ENV PYTHONUNBUFFERED=1
+
+RUN apt-get update && \
+    apt-get install -y libxml2-dev libxslt1-dev gdal-bin gnupg git-core && \
+    apt-get clean && \
+    rm -rf /var/cache/apt/* /var/lib/apt/lists/*
+
+RUN mkdir /app
+WORKDIR /app
+
+COPY ./requirements.txt /app/requirements.txt
+RUN pip install --upgrade pip && \
+    pip install --no-cache-dir -r requirements.txt
+
+COPY . /app
+
+# Copy Blackbox executables from builder stage
+COPY --from=builder /usr/local/bin/blackbox* /usr/local/bin/
+COPY --from=builder /usr/local/bin/_blackbox* /usr/local/bin/
+COPY --from=builder /usr/local/bin/_stack_lib.sh /usr/local/bin/
+
+ENTRYPOINT ["/app/docker-entrypoint.sh"]

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 DataMade LLC
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,26 @@
+scrapers-lametro
+=====================
+
+DataMade's source for municipal scrapers feeding [boardagendas.metro.net](https://boardagendas.metro.net).
+
+## Development
+
+### Making changes to this repository
+
+Make your changes to the scraper code here.
+
+Merge your PR to push to `main` and publish a `main` tag of the scraper image.
+
+To publish a `production` tag of the scraper image, sync the `main` branch with the
+`deploy` branch:
+
+```bash
+git push origin main:deploy
+```
+
+### Scheduling
+
+The LA Metro scrapers are scheduled via Airflow. The production Airflow instance
+is located at [la-metro-dashboard.datamade.us](https://la-metro-dashboard.datamade.us/).
+DataMade staff can find login credentials under the Metro support email in
+LastPass. The underlying code is in the [`datamade/la-metro-dashboard` repository](https://github.com/datamade/la-metro-dashboard).

docker-compose.councilmatic.yml

@@ -0,0 +1,26 @@
+version: '2.4'
+
+services:
+  scrapers:
+    image: scrapers-lametro
+    container_name: scrapers-lametro
+    build: .
+    stdin_open: true
+    tty: true
+    volumes:
+      - .:/app
+    environment:
+      # Populate the local Councilmatic database
+      DATABASE_URL: postgresql://postgres:postgres@postgres:5432/lametro
+      DJANGO_SETTINGS_MODULE: pupa.settings
+      OCD_DIVISION_CSV: "/app/lametro/divisions.csv"
+    command: pupa update lametro
+    # Connect the scraper container to the app network
+    networks:
+      - app_net
+
+networks:
+  # Define connection to the app's Docker network
+  app_net:
+    name: la-metro-councilmatic_default
+    external: true

docker-compose.yml

@@ -0,0 +1,38 @@
+version: '2.4'
+
+services:
+  scrapers:
+    image: scrapers-lametro
+    container_name: scrapers-lametro
+    build: .
+    stdin_open: true
+    tty: true
+    depends_on:
+      postgres:
+        condition: service_healthy
+    volumes:
+      - .:/app
+    environment:
+      DATABASE_URL: postgres://postgres:postgres@postgres/opencivicdata
+      DJANGO_SETTINGS_MODULE: pupa.settings
+      OCD_DIVISION_CSV: "/app/lametro/divisions.csv"
+    command: pupa update lametro
+
+  postgres:
+    container_name: scrapers-lametro-postgres
+    image: postgis/postgis:11-3.3
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    environment:
+      POSTGRES_DB: opencivicdata
+      POSTGRES_PASSWORD: postgres
+    volumes:
+      - scrapers-lametro-db-data:/var/lib/postgresql/data
+    ports:
+      - 32001:5432
+
+volumes:
+  scrapers-lametro-db-data:

docker-entrypoint.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+set -e
+
+pupa dbinit us
+
+exec "$@"

keyrings/live/blackbox-admins.txt

@@ -0,0 +1,3 @@
+datamade@scrapers-us-municipal
+[email protected]
+[email protected]

keyrings/live/blackbox-files.txt

@@ -0,0 +1 @@
+lametro/secrets.py