Skip to content

Commit

Permalink
Passer sur une distribution Debian comme base d'image pour la base de…
Browse files Browse the repository at this point in the history
… données (#3216)

## Linked issues

- Resolve #3215
- Resolve #4010
- Resolve #4011
  • Loading branch information
louptheron authored Jan 7, 2025
2 parents f2954ee + a8c63b6 commit c02a565
Show file tree
Hide file tree
Showing 11 changed files with 215 additions and 92 deletions.
2 changes: 2 additions & 0 deletions .github/workflows/cicd.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,10 @@ on:
- "**.tsx"
- "**.yml"
- ".github/workflows/cicd.yml"
- ".github/workflows/database.yml"
- "infra/configurations/**.properties"
- "infra/docker/app/**"
- "infra/docker/database/**"
- "**.sql"
- "backend/pom.xml"
- "backend/build.gradle.kts"
Expand Down
19 changes: 6 additions & 13 deletions .github/workflows/database.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,14 @@ jobs:
image_version: "pg11-ts2.3.1-postgis3.3.4"
- pg_major: "13"
timescaledb_version: "2.14.2"
postgis_version: "3.4.2"
postgis_version: "3.5.1"
dockerfile: "pg13_16.Dockerfile"
image_version: "pg13-ts2.14.2-postgis3.4.2"
image_version: "pg13-ts2.14.2-postgis3.5.1"
- pg_major: "16"
timescaledb_version: "2.14.2"
postgis_version: "3.4.2"
postgis_version: "3.5.1"
dockerfile: "pg13_16.Dockerfile"
image_version: "pg16-ts2.14.2-postgis3.4.2"
image_version: "pg16-ts2.14.2-postgis3.5.1"
permissions:
contents: read
packages: write
Expand Down Expand Up @@ -72,20 +72,13 @@ jobs:
strategy:
matrix:
include:
- distribution: "buster"
from_pg_major: "11"
to_pg_major: "13"
timescaledb_version: "2.3.1"
postgis_version: "3.3.4"
dockerfile: "upgrade_pg.Dockerfile"
image_version: "pg11_to_pg13-ts2.3.1-postgis3.3.4"
- distribution: "bookworm"
from_pg_major: "13"
to_pg_major: "16"
timescaledb_version: "2.14.2"
postgis_version: "3.4.2"
postgis_version: "3.5.1"
dockerfile: "upgrade_pg.Dockerfile"
image_version: "pg13_to_pg16-ts2.14.2-postgis3.4.2"
image_version: "pg13_to_pg16-ts2.14.2-postgis3.5.1"

permissions:
contents: read
Expand Down
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,7 @@ upgrade-postgres-11-to-13:
-v $(PG_13_DATA_VOLUME_NAME):/var/lib/postgresql/13/data \
ghcr.io/mtes-mct/monitorfish/monitorfish-database-upgrade:pg11_to_pg13-ts2.3.1-postgis3.3.4 -O "-c timescaledb.restoring='on'" -O "-c shared_preload_libraries=timescaledb";

upgrade-postgres-13-to-16-dev:
upgrade-postgres-13-to-16:
docker run --rm \
-v $(PG_13_DATA_VOLUME_NAME):/var/lib/postgresql/13/data \
-v $(PG_16_DATA_VOLUME_NAME):/var/lib/postgresql/16/data \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ context('BackOffice > Fleet Segments Table', () => {
segment: 'ATL036',
segmentName: 'All Trawls 45',
targetSpecies: ['COD'],
year: 2024
year: currentYear
})
})

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ context('Vessel sidebar controls tab', () => {

// Because date is set as "NOW - 1 YEAR - 1 MONTH", we might be in the first month of the year
if (currentMonth === 0) {
cy.get(`ul > div[data-cy="vessel-control-years"] > div > div[title="${currentYear - 1}"]`).contains(
cy.get(`[data-cy="vessel-control-years"] > div > div[title="${currentYear - 1}"]`).contains(
"1 contrôle, pas d'infraction"
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -528,7 +528,7 @@ describe('isForbiddenPeriod', () => {
})

// When
const currentDate = customDayjs().utc().set('month', 1).set('date', 1) // Monday
const currentDate = customDayjs().utc().set('month', 1).set('day', 1) // Monday
expect(isForbiddenPeriod(feature, currentDate)).toEqual(false)
})

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
#!/bin/bash

create_sql=`mktemp`

# Checks to support bitnami image with same scripts so they stay in sync
if [ ! -z "${BITNAMI_APP_NAME:-}" ]; then
if [ -z "${POSTGRES_USER:-}" ]; then
POSTGRES_USER=postgres
fi

if [ -z "${POSTGRESQL_PASSWORD:-}" ]; then
POSTGRESQL_PASSWORD=${POSTGRESQL_POSTGRES_PASSWORD}
fi

if [ -z "${POSTGRES_DB:-}" ]; then
POSTGRES_DB=${POSTGRESQL_DATABASE}
fi

if [ -z "${PGDATA:-}" ]; then
PGDATA=${POSTGRESQL_DATA_DIR}
fi
fi

if [ -z "${POSTGRESQL_CONF_DIR:-}" ]; then
POSTGRESQL_CONF_DIR=${PGDATA}
fi

cat <<EOF >${create_sql}
CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE;
EOF

TS_TELEMETRY='basic'
if [ "${TIMESCALEDB_TELEMETRY:-}" == "off" ]; then
TS_TELEMETRY='off'

# We delete the job as well to ensure that we do not spam the
# log with other messages related to the Telemetry job.
cat <<EOF >>${create_sql}
SELECT alter_job(1,scheduled:=false);
EOF
fi

echo "timescaledb.telemetry_level=${TS_TELEMETRY}" >> ${POSTGRESQL_CONF_DIR}/postgresql.conf

if [ -z "${POSTGRESQL_PASSWORD:-}" ]; then
POSTGRESQL_PASSWORD=${POSTGRES_PASSWORD:-}
fi
export PGPASSWORD="$POSTGRESQL_PASSWORD"

# create extension timescaledb in initial databases
psql -U "${POSTGRES_USER}" postgres -f ${create_sql}
psql -U "${POSTGRES_USER}" template1 -f ${create_sql}

if [ "${POSTGRES_DB:-postgres}" != 'postgres' ]; then
psql -U "${POSTGRES_USER}" "${POSTGRES_DB}" -f ${create_sql}
fi
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
#!/bin/bash

NO_TS_TUNE=${NO_TS_TUNE:-""}
TS_TUNE_MEMORY=${TS_TUNE_MEMORY:-""}
TS_TUNE_NUM_CPUS=${TS_TUNE_NUM_CPUS:-""}
TS_TUNE_MAX_CONNS=${TS_TUNE_MAX_CONNS:-""}
TS_TUNE_MAX_BG_WORKERS=${TS_TUNE_MAX_BG_WORKERS:-""}

if [ ! -z "${NO_TS_TUNE:-}" ]; then
# The user has explicitly requested not to run timescaledb-tune; exit this script
exit 0
fi


if [ -z "${POSTGRESQL_CONF_DIR:-}" ]; then
POSTGRESQL_CONF_DIR=${PGDATA}
fi

if [ -z "${TS_TUNE_MEMORY:-}" ]; then
# See if we can get the container's total allocated memory from the cgroups metadata
if [ -f /sys/fs/cgroup/memory/memory.limit_in_bytes ]; then
TS_TUNE_MEMORY=$(cat /sys/fs/cgroup/memory/memory.limit_in_bytes)

if [ "${TS_TUNE_MEMORY}" = "18446744073709551615" ]; then
# Bash seems to error out for numbers greater than signed 64-bit,
# so if the value of limit_in_bytes is the 64-bit UNSIGNED max value
# we should just bail out and hope timescaledb-tune can figure this
# out. If we don't, the next comparison is likely going to fail
# or it might store a negative value which will crash later.
TS_TUNE_MEMORY=""
fi

FREE_KB=$(grep MemTotal: /proc/meminfo | awk '{print $2}')
FREE_BYTES=$(( ${FREE_KB} * 1024 ))
if [ ${TS_TUNE_MEMORY} -gt ${FREE_BYTES} ]; then
# Something weird is going on if the cgroups memory limit exceeds the total available
# amount of system memory reported by "free", which is the total amount of memory available on the host.
# Most likely, it is this issue: https://github.com/moby/moby/issues/18087 (if no limit is
# set, the max limit is set to the max 64 bit integer). In this case, we just leave
# TS_TUNE_MEMORY blank and let timescaledb-tune derive the memory itself using syscalls.
TS_TUNE_MEMORY=""
else
# Convert the bytes to MB so it plays nicely with timescaledb-tune
TS_TUNE_MEMORY="$(echo ${TS_TUNE_MEMORY} | awk '{print int($1 / 1024 / 1024)}')MB"
fi
fi
fi

if [ -z "${TS_TUNE_NUM_CPUS:-}" ]; then
# See if we can get the container's available CPUs from the cgroups metadata
if [ -f /sys/fs/cgroup/cpuset/cpuset.cpus ]; then
TS_TUNE_NUM_CPUS=$(cat /sys/fs/cgroup/cpuset/cpuset.cpus)
if [[ ${TS_TUNE_NUM_CPUS} == *-* ]]; then
# The CPU limits have been defined as a range (e.g., 0-3 for 4 CPUs). Subtract them and add 1
# to convert the range to the number of CPUs.
TS_TUNE_NUM_CPUS=$(echo ${TS_TUNE_NUM_CPUS} | tr "-" " " | awk '{print ($2 - $1) + 1}')
elif [[ ${TS_TUNE_NUM_CPUS} == *,* ]]; then
# The CPU limits have been defined as a comma separated list (e.g., 0,1,2,3 for 4 CPUs). Count each CPU
TS_TUNE_NUM_CPUS=$(echo ${TS_TUNE_NUM_CPUS} | tr "," "\n" | wc -l)
elif [ $(echo -n ${TS_TUNE_NUM_CPUS} | wc -c) -eq 1 ]; then
# The CPU limit has been defined as a single numbered CPU. In this case the CPU limit is 1
# regardless of what that number is
TS_TUNE_NUM_CPUS=1
fi
fi
fi

if [ ! -z "${TS_TUNE_MEMORY:-}" ]; then
TS_TUNE_MEMORY_FLAGS=--memory="${TS_TUNE_MEMORY}"
fi

if [ ! -z "${TS_TUNE_NUM_CPUS:-}" ]; then
TS_TUNE_NUM_CPUS_FLAGS=--cpus=${TS_TUNE_NUM_CPUS}
fi

if [ ! -z "${TS_TUNE_MAX_CONNS:-}" ]; then
TS_TUNE_MAX_CONNS_FLAGS=--max-conns=${TS_TUNE_MAX_CONNS}
fi

if [ ! -z "${TS_TUNE_MAX_BG_WORKERS:-}" ]; then
TS_TUNE_MAX_BG_WORKERS_FLAGS=--max-bg-workers=${TS_TUNE_MAX_BG_WORKERS}
fi

if [ ! -z "${PG_MAJOR}" ]; then
TS_TUNE_PG_VERSION=--pg-version=${PG_MAJOR}
fi

/usr/local/bin/timescaledb-tune --quiet --yes --conf-path="${POSTGRESQL_CONF_DIR}/postgresql.conf" ${TS_TUNE_MEMORY_FLAGS} ${TS_TUNE_NUM_CPUS_FLAGS} ${TS_TUNE_MAX_CONNS_FLAGS} ${TS_TUNE_MAX_BG_WORKERS_FLAGS} ${TS_TUNE_PG_VERSION}
128 changes: 54 additions & 74 deletions infra/docker/database/pg13_16.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -2,79 +2,59 @@ ARG PG_MAJOR
ARG TIMESCALEDB_VERSION
ARG POSTGIS_VERSION

FROM timescale/timescaledb:${TIMESCALEDB_VERSION}-pg${PG_MAJOR}
############################
# Build tools binaries in separate image
############################
ARG GO_VERSION=1.22.4
FROM golang:${GO_VERSION}-alpine AS tools

ENV TOOLS_VERSION 0.8.1

Check warning on line 11 in infra/docker/database/pg13_16.Dockerfile

View workflow job for this annotation

GitHub Actions / Build database image (13, 2.14.2, 3.5.1, pg13_16.Dockerfile, pg13-ts2.14.2-postgis3.5.1)

Legacy key/value format with whitespace separator should not be used

LegacyKeyValueFormat: "ENV key=value" should be used instead of legacy "ENV key value" format More info: https://docs.docker.com/go/dockerfile/rule/legacy-key-value-format/

Check warning on line 11 in infra/docker/database/pg13_16.Dockerfile

View workflow job for this annotation

GitHub Actions / Build database image (16, 2.14.2, 3.5.1, pg13_16.Dockerfile, pg16-ts2.14.2-postgis3.5.1)

Legacy key/value format with whitespace separator should not be used

LegacyKeyValueFormat: "ENV key=value" should be used instead of legacy "ENV key value" format More info: https://docs.docker.com/go/dockerfile/rule/legacy-key-value-format/

RUN apk update && apk add --no-cache git gcc musl-dev \
&& go install github.com/timescale/timescaledb-tune/cmd/timescaledb-tune@latest \
&& go install github.com/timescale/timescaledb-parallel-copy/cmd/timescaledb-parallel-copy@latest

############################
# Now build image and copy in tools
############################
FROM postgres:"$PG_MAJOR"-bookworm

Check warning on line 20 in infra/docker/database/pg13_16.Dockerfile

View workflow job for this annotation

GitHub Actions / Build database image (13, 2.14.2, 3.5.1, pg13_16.Dockerfile, pg13-ts2.14.2-postgis3.5.1)

Default value for global ARG results in an empty or invalid base image name

InvalidDefaultArgInFrom: Default value for ARG postgres:"$PG_MAJOR"-bookworm results in empty or invalid base image name More info: https://docs.docker.com/go/dockerfile/rule/invalid-default-arg-in-from/

Check warning on line 20 in infra/docker/database/pg13_16.Dockerfile

View workflow job for this annotation

GitHub Actions / Build database image (16, 2.14.2, 3.5.1, pg13_16.Dockerfile, pg16-ts2.14.2-postgis3.5.1)

Default value for global ARG results in an empty or invalid base image name

InvalidDefaultArgInFrom: Default value for ARG postgres:"$PG_MAJOR"-bookworm results in empty or invalid base image name More info: https://docs.docker.com/go/dockerfile/rule/invalid-default-arg-in-from/
ARG PG_MAJOR
ARG TIMESCALEDB_VERSION
ARG POSTGIS_VERSION

RUN set -eux \
&& apk add --no-cache --virtual .fetch-deps \
ca-certificates \
openssl \
tar \
\
&& wget -O postgis.tar.gz "https://github.com/postgis/postgis/archive/${POSTGIS_VERSION}.tar.gz" \
&& mkdir -p /usr/src/postgis \
&& tar \
--extract \
--file postgis.tar.gz \
--directory /usr/src/postgis \
--strip-components 1 \
&& rm postgis.tar.gz \
\
&& apk add --no-cache --virtual .build-deps \
\
gdal-dev \
geos-dev \
proj-dev \
proj-util \
sfcgal-dev \
\
# The upstream variable, '$DOCKER_PG_LLVM_DEPS' contains
# the correct versions of 'llvm-dev' and 'clang' for the current version of PostgreSQL.
# This improvement has been discussed in https://github.com/docker-library/postgres/pull/1077
$DOCKER_PG_LLVM_DEPS \
\
autoconf \
automake \
cunit-dev \
file \
g++ \
gcc \
gettext-dev \
git \
json-c-dev \
libtool \
libxml2-dev \
make \
pcre2-dev \
perl \
protobuf-c-dev \
\
# build PostGIS - with Link Time Optimization (LTO) enabled
&& cd /usr/src/postgis \
&& gettextize \
&& ./autogen.sh \
&& ./configure \
--enable-lto \
&& make -j$(nproc) \
&& make install \
\
# add .postgis-rundeps
&& apk add --no-cache --virtual .postgis-rundeps \
\
gdal \
geos \
proj \
sfcgal \
\
json-c \
libstdc++ \
pcre2 \
protobuf-c \
\
# ca-certificates: for accessing remote raster files
# fix https://github.com/postgis/docker-postgis/issues/307
ca-certificates \
# clean
&& cd / \
&& rm -rf /usr/src/postgis \
&& apk del .fetch-deps .build-deps \
COPY infra/docker/database/docker-entrypoint-initdb.d/* /docker-entrypoint-initdb.d/
COPY --from=tools /go/bin/* /usr/local/bin/

RUN \
apt-get update && \
apt-get install -y \
wget \
lsb-release && \
rm -rf /var/lib/apt/lists/*

# Install TimescaleDB extension
RUN echo "deb https://packagecloud.io/timescale/timescaledb/debian/ $(lsb_release -c -s) main"
RUN echo "deb https://packagecloud.io/timescale/timescaledb/debian/ $(lsb_release -c -s) main" > /etc/apt/sources.list.d/timescaledb.list
RUN wget --quiet -O - https://packagecloud.io/timescale/timescaledb/gpgkey | apt-key add -

RUN \
TIMESCALEDB_MAJOR=$(echo "$TIMESCALEDB_VERSION" | cut -c1) && \
apt-get update && \
apt-get install -y \
timescaledb-"$TIMESCALEDB_MAJOR"-postgresql-"$PG_MAJOR"="$TIMESCALEDB_VERSION"* \
timescaledb-"$TIMESCALEDB_MAJOR"-loader-postgresql-"$PG_MAJOR"="$TIMESCALEDB_VERSION"* && \
rm -rf /var/lib/apt/lists/*

RUN for file in $(find /usr/share/postgresql -name 'postgresql.conf.sample'); do \
# We want timescaledb to be loaded in this image by every created cluster
sed -r -i "s/[#]*\s*(shared_preload_libraries)\s*=\s*'(.*)'/\1 = 'timescaledb,\2'/;s/,'/'/" "$file"; \
done

# Install PostGIS extension
RUN \
POSTGIS_MAJOR=$(echo "$POSTGIS_VERSION" | cut -c1) && \
apt update && \
apt install -y --no-install-recommends \
postgresql-"$PG_MAJOR"-postgis-"$POSTGIS_MAJOR"="$POSTGIS_VERSION"* \
postgresql-"$PG_MAJOR"-postgis-"$POSTGIS_MAJOR"-scripts && \
rm -rf /var/lib/apt/lists/*
5 changes: 5 additions & 0 deletions infra/docker/database/upgrade_pg.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,11 @@ RUN \
apt-get install -y timescaledb-$TIMESCALEDB_MAJOR-postgresql-$FROM_PG_MAJOR=$TIMESCALEDB_VERSION* timescaledb-$TIMESCALEDB_MAJOR-loader-postgresql-$FROM_PG_MAJOR=$TIMESCALEDB_VERSION* && \
apt-get install -y timescaledb-$TIMESCALEDB_MAJOR-postgresql-$TO_PG_MAJOR=$TIMESCALEDB_VERSION* timescaledb-$TIMESCALEDB_MAJOR-loader-postgresql-$TO_PG_MAJOR=$TIMESCALEDB_VERSION*

RUN for file in $(find /usr/share/postgresql -name 'postgresql.conf.sample'); do \
# We want timescaledb to be loaded in this image by every created cluster
sed -r -i "s/[#]*\s*(shared_preload_libraries)\s*=\s*'(.*)'/\1 = 'timescaledb,\2'/;s/,'/'/" "$file"; \
done

# Install PostGIS extension in both versions of Postgres
RUN apt update
RUN \
Expand Down
1 change: 0 additions & 1 deletion infra/remote/docker-compose.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
version: "3"
services:
db:
image: ghcr.io/mtes-mct/monitorfish/monitorfish-database:$DATABASE_VERSION
Expand Down

0 comments on commit c02a565

Please sign in to comment.