diff --git a/.gitignore b/.gitignore index b5bd985..b9a408d 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ cdk.out .env .ipynb_checkpoints data/ + +.pgdata \ No newline at end of file diff --git a/README.md b/README.md index f4d6de4..d4dc039 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,16 @@ To add append to a table: `ogr2ogr -f "PostgreSQL" PG:"host=HOST dbname=DBNAME user=USER password=PASSWORD" "LargeFires_2012-2020.gpkg" -nln fire_boundaries2 -append` +## Docker for local development + +To run locally on docker use: + +`docker compose up` + +To append to the docker database: + +`ogr2ogr -f "PostgreSQL" PG:"host=localhost dbname=postgis user=username password=password" "LargeFires_2012-2020.gpkg" -nln fire_boundaries2 -append` + ## Commiting We use this action to create tags https://github.com/mathieudutour/github-tag-action diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..f2cb0c9 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,38 @@ +version: '3' + +services: + app: + build: + dockerfile: local/Dockerfile + environment: + - HOST=0.0.0.0 + - PORT=8081 + - PYTHONWARNINGS=ignore + - POSTGRES_USER=username + - POSTGRES_PASS=password + - POSTGRES_DBNAME=postgis + - POSTGRES_HOST=database + - POSTGRES_PORT=5432 + - DEBUG=TRUE + ports: + - "${MY_DOCKER_IP:-127.0.0.1}:8081:8081" + depends_on: + - database + command: + bash -c "bash /tmp/scripts/wait-for-it.sh database:5432 --timeout=0 + && uvicorn app.handler:app --host 0.0.0.0 --port 8081 --reload" + volumes: + - ./scripts:/tmp/scripts + - ./stack/app:/tmp/app + database: + build: + dockerfile: local/Dockerfile.db + environment: + - POSTGRES_USER=username + - POSTGRES_PASSWORD=password + - POSTGRES_DB=postgis + ports: + - "5432:5432" + command: postgres -N 500 + volumes: + - ./.pgdata:/var/lib/postgresql/data \ No newline at end of file diff --git a/local/Dockerfile b/local/Dockerfile new file mode 100644 index 0000000..405638b --- /dev/null +++ b/local/Dockerfile @@ -0,0 +1,11 @@ +ARG PYTHON_VERSION=3.10 + +FROM ghcr.io/vincentsarago/uvicorn-gunicorn:${PYTHON_VERSION} + +WORKDIR /tmp + +COPY stack/app/requirements.txt requirements.txt +RUN pip install -r requirements.txt + +# ENV MODULE_NAME app.handler +# ENV VARIABLE_NAME app \ No newline at end of file diff --git a/local/Dockerfile.db b/local/Dockerfile.db new file mode 100644 index 0000000..a030b57 --- /dev/null +++ b/local/Dockerfile.db @@ -0,0 +1,4 @@ +FROM ghcr.io/vincentsarago/postgis:14-3.3 + +#sample data +# COPY data /docker-entrypoint-initdb.d/ \ No newline at end of file diff --git a/scripts/wait-for-it.sh b/scripts/wait-for-it.sh new file mode 100755 index 0000000..7593a22 --- /dev/null +++ b/scripts/wait-for-it.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash +# Use this script to test if a given TCP host/port are available + +###################################################### +# Copied from https://github.com/vishnubob/wait-for-it +###################################################### + +WAITFORIT_cmdname=${0##*/} + +echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + else + echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" + fi + WAITFORIT_start_ts=$(date +%s) + while : + do + if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then + nc -z $WAITFORIT_HOST $WAITFORIT_PORT + WAITFORIT_result=$? + else + (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 + WAITFORIT_result=$? + fi + if [[ $WAITFORIT_result -eq 0 ]]; then + WAITFORIT_end_ts=$(date +%s) + echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" + break + fi + sleep 1 + done + return $WAITFORIT_result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $WAITFORIT_QUIET -eq 1 ]]; then + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + else + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + fi + WAITFORIT_PID=$! + trap "kill -INT -$WAITFORIT_PID" INT + wait $WAITFORIT_PID + WAITFORIT_RESULT=$? + if [[ $WAITFORIT_RESULT -ne 0 ]]; then + echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + fi + return $WAITFORIT_RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + WAITFORIT_hostport=(${1//:/ }) + WAITFORIT_HOST=${WAITFORIT_hostport[0]} + WAITFORIT_PORT=${WAITFORIT_hostport[1]} + shift 1 + ;; + --child) + WAITFORIT_CHILD=1 + shift 1 + ;; + -q | --quiet) + WAITFORIT_QUIET=1 + shift 1 + ;; + -s | --strict) + WAITFORIT_STRICT=1 + shift 1 + ;; + -h) + WAITFORIT_HOST="$2" + if [[ $WAITFORIT_HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + WAITFORIT_HOST="${1#*=}" + shift 1 + ;; + -p) + WAITFORIT_PORT="$2" + if [[ $WAITFORIT_PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + WAITFORIT_PORT="${1#*=}" + shift 1 + ;; + -t) + WAITFORIT_TIMEOUT="$2" + if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + WAITFORIT_TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + WAITFORIT_CLI=("$@") + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} +WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} +WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} +WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} + +# Check to see if timeout is from busybox? +WAITFORIT_TIMEOUT_PATH=$(type -p timeout) +WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) + +WAITFORIT_BUSYTIMEFLAG="" +if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then + WAITFORIT_ISBUSY=1 + # Check if busybox timeout uses -t flag + # (recent Alpine versions don't support -t anymore) + if timeout &>/dev/stdout | grep -q -e '-t '; then + WAITFORIT_BUSYTIMEFLAG="-t" + fi +else + WAITFORIT_ISBUSY=0 +fi + +if [[ $WAITFORIT_CHILD -gt 0 ]]; then + wait_for + WAITFORIT_RESULT=$? + exit $WAITFORIT_RESULT +else + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + wait_for_wrapper + WAITFORIT_RESULT=$? + else + wait_for + WAITFORIT_RESULT=$? + fi +fi + +if [[ $WAITFORIT_CLI != "" ]]; then + if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then + echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" + exit $WAITFORIT_RESULT + fi + exec "${WAITFORIT_CLI[@]}" +else + exit $WAITFORIT_RESULT +fi \ No newline at end of file diff --git a/stack/app.py b/stack/app.py index 2a6f1b4..018992d 100644 --- a/stack/app.py +++ b/stack/app.py @@ -155,7 +155,7 @@ def __init__( setup_db = BootstrappedDb( self, - "STAC DB for EIS Fires", + "Features DB for EIS Fires", db=db, new_dbname=db_settings.dbname, new_username=db_settings.user, diff --git a/stack/config.py b/stack/config.py index d7c032f..f6ffcdd 100644 --- a/stack/config.py +++ b/stack/config.py @@ -9,7 +9,7 @@ class APISettings(pydantic.BaseSettings): """Application settings""" name: str = "tifeatures-timvt" - stage: str = "staging" + stage: str = "dev" owner: Optional[str] client: Optional[str]