Upgrade to docker compose v2 (deephaven#5322)

This upgrades the gradle-docker-compose-plugin to support docker compose v2 by default. The important release notes are https://github.com/avast/gradle-docker-compose-plugin/releases/tag/0.17.0, although we are updating to the latest 0.17.6. Some manual configuration of dockerCompose was needed to workaround avast/gradle-docker-compose-plugin#393.

Additionally, this upgrades all of our docker-compose.yml files to be proper v2 files by removing the version. The docker compose version is only relevant for docker-compose (v1). See https://docs.docker.com/compose/intro/history/ for clarification. This was discovered based on the warning from the `docker compose` process, which would otherwise warn:

```
> Task :Integrations:composeBuild
time="2024-04-03T12:09:28-07:00" level=warning msg="/home/devin/dev/deephaven/deephaven-core/Integrations/docker-compose.yml: `version` is obsolete"
```

Fixes deephaven#5320

Integrations/build.gradle

@@ -1,3 +1,6 @@
+import java.nio.charset.StandardCharsets
+import java.security.MessageDigest
+
 plugins {
     id 'com.bmuschko.docker-remote-api'
     id 'io.deephaven.project.register'
@@ -103,8 +106,17 @@ def runInDocker = { String name, String sourcePath, List<String> command, Closur
     }
 }
 
-// Using Integrations/docker-compose.yml, this will start redpanda and wait until it is running before letting tests proceed.
+// Using Integrations/docker-compose.yml, this will start all the containers and wait until the exposed ports are ready
 dockerCompose {
+    // To workaround errors encountered when dockerCompose auto-inherits the projectName (it uses "-" and "_" next to
+    // each other, which violates docker naming conventions), we can explicitly set a projectName.
+    // https://github.com/avast/gradle-docker-compose-plugin/issues/393
+    // https://github.com/avast/gradle-docker-compose-plugin/issues/372#issuecomment-1403264410
+    // Additionally, this preserves the prefix logic that dockerCompose chooses to use based on the absolutePath of the
+    // project. This may be beneficial in workflows where the user has checked out multiple repositories and wants to
+    // ensure docker names do not collide.
+    def projectSafePrefix = MessageDigest.getInstance("MD5").digest(project.projectDir.absolutePath.toString().getBytes(StandardCharsets.UTF_8)).encodeHex().toString()
+    projectName = "${projectSafePrefix}_${project.name.replace('.', '_')}"
     waitForTcpPortsTimeout = Duration.ofMinutes(2)
 }
 

Integrations/docker-compose.yml

@@ -1,7 +1,5 @@
 # test-oriented local docker compose file to run redpanda and postgres for testing
 
-version: "3.4"
-
 services:
   redpanda:
     extends:

authentication/example-providers/oidc/README.md

@@ -11,9 +11,9 @@ server component is designed using [Pac4j](https://www.pac4j.org/), making it fl
 general OpenID Connect services, but also other kinds of single-sign on services like SAML, OAuth, etc.
 
 ## Example Keycloak setup
-In this directory, run `docker-compose up` to start Keycloak and database for it. When the container first starts up, a
+In this directory, run `docker compose up` to start Keycloak and database for it. When the container first starts up, a
 [JSON file](deephaven_realm.json) configures a realm and a client that can access it, and two users: `admin`/`admin` and
-`user`/`user`. This docker-compose configuration is designed to be fast to start and ephemeral, and is not meant for
+`user`/`user`. This docker compose configuration is designed to be fast to start and ephemeral, and is not meant for
 production use.
 
 After that has run, there will be a docker container running with port 6060 exposed on localhost. The account to

authentication/example-providers/oidc/docker-compose.yml

@@ -1,5 +1,3 @@
-version: "3.4"
-
 services:
   db:
     image: postgres:14.2

authentication/example-providers/sql-username-password/README.md

@@ -6,8 +6,8 @@ to follow best practices in storing secure information like the password itself,
 rudimentary.
 
 ## Example database setup
-In this directory, run `docker-compose up` to start a SQL database. The `init-users.sql` file will be used to initialize
-this database, which has one user, named `admin`, with a password of `p@ssw0rd`. This docker-compose configuration is
+In this directory, run `docker compose up` to start a SQL database. The `init-users.sql` file will be used to initialize
+this database, which has one user, named `admin`, with a password of `p@ssw0rd`. This docker compose configuration is
 designed to be fast to start and ephemeral, and is not meant for production use.
 
 After that has run, there will be a docker container running with port 5432 exposed on localhost. The password for the

authentication/example-providers/sql-username-password/docker-compose.yml

@@ -1,5 +1,3 @@
-version: "3.4"
-
 services:
   postgres:
     image: postgres:15.1

build.gradle

@@ -64,7 +64,7 @@ tasks.register('nightly') {
 
 tasks.register('prepareCompose') {
     it.group 'Deephaven lifecycle'
-    it.description 'A lifecycle task that prepares prerequisites for local docker-compose builds'
+    it.description 'A lifecycle task that prepares prerequisites for local docker compose builds'
     it.dependsOn project(':docker-server-jetty').tasks.findByName('buildDocker-server-jetty')
 }
 

buildSrc/build.gradle

@@ -26,7 +26,7 @@ dependencies {
         because('needed by plugin com.bmuschko.docker-remote-api')
     }
 
-    implementation('com.avast.gradle:gradle-docker-compose-plugin:0.16.11') {
+    implementation('com.avast.gradle:gradle-docker-compose-plugin:0.17.6') {
         because('needed by plugin com.avast.gradle.docker-compose')
     }
 

containers/groovy-examples-redpanda/docker-compose.yml

@@ -1,5 +1,3 @@
-version: "3.4"
-
 services:
   deephaven:
     image: ghcr.io/deephaven/server-slim:${VERSION:-latest}

containers/groovy-examples/docker-compose.yml

@@ -1,5 +1,3 @@
-version: "3.4"
-
 services:
   deephaven:
     image: ghcr.io/deephaven/server-slim:${VERSION:-latest}

containers/groovy-redpanda/docker-compose.yml

@@ -1,5 +1,3 @@
-version: "3.4"
-
 services:
   deephaven:
     image: ghcr.io/deephaven/server-slim:${VERSION:-latest}

containers/groovy/docker-compose.yml

@@ -1,5 +1,3 @@
-version: "3.4"
-
 services:
   deephaven:
     image: ghcr.io/deephaven/server-slim:${VERSION:-latest}

containers/python-examples-redpanda/docker-compose.yml

@@ -1,5 +1,3 @@
-version: "3.4"
-
 services:
   deephaven:
     image: ghcr.io/deephaven/server:${VERSION:-latest}

containers/python-examples/NLTK/docker-compose.yml

@@ -1,5 +1,3 @@
-version: "3.4"
-
 services:
   deephaven:
     image: ghcr.io/deephaven/server-nltk:${VERSION:-latest}

containers/python-examples/PyTorch/docker-compose.yml

@@ -1,5 +1,3 @@
-version: "3.4"
-
 services:
   deephaven:
     image: ghcr.io/deephaven/server-pytorch:${VERSION:-latest}

containers/python-examples/SciKit-Learn/docker-compose.yml

@@ -1,5 +1,3 @@
-version: "3.4"
-
 services:
   deephaven:
     image: ghcr.io/deephaven/server-sklearn:${VERSION:-latest}

containers/python-examples/TensorFlow/docker-compose.yml

@@ -1,5 +1,3 @@
-version: "3.4"
-
 services:
   deephaven:
     image: ghcr.io/deephaven/server-tensorflow:${VERSION:-latest}

containers/python-examples/base/docker-compose.yml

@@ -1,5 +1,3 @@
-version: "3.4"
-
 services:
   deephaven:
     image: ghcr.io/deephaven/server:${VERSION:-latest}

containers/python-redpanda/docker-compose.yml

@@ -1,5 +1,3 @@
-version: "3.4"
-
 services:
   deephaven:
     image: ghcr.io/deephaven/server:${VERSION:-latest}

containers/python/NLTK/docker-compose.yml

@@ -1,5 +1,3 @@
-version: "3.4"
-
 services:
   deephaven:
     image: ghcr.io/deephaven/server-nltk:${VERSION:-latest}

containers/python/PyTorch/docker-compose.yml

@@ -1,5 +1,3 @@
-version: "3.4"
-
 services:
   deephaven:
     image: ghcr.io/deephaven/server-pytorch:${VERSION:-latest}

containers/python/SciKit-Learn/docker-compose.yml

@@ -1,5 +1,3 @@
-version: "3.4"
-
 services:
   deephaven:
     image: ghcr.io/deephaven/server-sklearn:${VERSION:-latest}

containers/python/TensorFlow/docker-compose.yml

@@ -1,5 +1,3 @@
-version: "3.4"
-
 services:
   deephaven:
     image: ghcr.io/deephaven/server-tensorflow:${VERSION:-latest}

containers/python/base/docker-compose.yml

@@ -1,5 +1,3 @@
-version: "3.4"
-
 services:
   deephaven:
     image: ghcr.io/deephaven/server:${VERSION:-latest}

debezium/demo/README.md

@@ -34,12 +34,12 @@ and ensure you can launch a regular version of Deephaven
 as per the instructions there.  Once that works, stop that
 Deephaven instance, and continue below.
 
-Start docker-compose with the compose file in this
+Start docker compose with the compose file in this
 directory:
 
 ```
 cd debezium/demo
-docker-compose up --build
+docker compose up --build
 ```
 
 Then start a Deephaven web console (will be in python mode

debezium/demo/docker-compose.yml

@@ -1,8 +1,6 @@
 # docker compose file to run the debezium-kafka ecommerce demo
 # with DHC ticking tables and dashboard
 
-version: '3.4'
-
 services:
   server:
     extends:

debezium/docker-compose-debezium-common.yml

@@ -1,8 +1,6 @@
 # docker compose file to run the debezium-kafka ecommerce demo
 # with DHC ticking tables and dashboard
 
-version: '3.4'
-
 services:
   redpanda:
     image: docker.redpanda.com/vectorized/redpanda:${REDPANDA_VERSION}

debezium/perf/README.md

@@ -159,7 +159,7 @@ can make them execute their respective demo scripts.
   see the tables refreshing.
 * To load the Materialize script, run the
   materialized command line interface (cli) via:
-  `docker-compose run mzcli`
+  `docker compose run mzcli`
   (this command needs to be executed from the `perf`
    directory to work).
   Once in the materialize cli, run:

debezium/perf/docker-compose.yml

@@ -1,8 +1,6 @@
 # docker compose file to run the debezium-kafka ecommerce demo
 # with DHC ticking tables and dashboard
 
-version: '3.4'
-
 services:
   server:
     extends:

debezium/perf/mz_run_demo.sh

@@ -2,4 +2,4 @@
 
 set -eu
 
-exec docker-compose run -T mzcli -f /scripts/demo.sql
+exec docker compose run -T mzcli -f /scripts/demo.sql

debezium/perf/mz_sample_dt.sh

@@ -20,6 +20,6 @@ done
 EOF
 )
 
-(nohup docker-compose run -T --entrypoint /bin/bash mzcli -c "$SCRIPT" < /dev/null >& $OUT &)
+(nohup docker compose run -T --entrypoint /bin/bash mzcli -c "$SCRIPT" < /dev/null >& $OUT &)
 
 exit 0

debezium/perf/start_perf_run.sh

@@ -30,17 +30,17 @@ LOG="$OUT_DIR/start.log"
 rm -f $LOG
 
 if [ "$engine" = "mz" ]; then
-    docker-compose up -d mysql redpanda debezium loadgen materialized mzcli >> $LOG 2>&1
+    docker compose up -d mysql redpanda debezium loadgen materialized mzcli >> $LOG 2>&1
 elif [ "$engine" = "dh" ]; then
-    docker-compose up -d mysql redpanda debezium loadgen server grpc-proxy envoy web >> $LOG 2>&1
+    docker compose up -d mysql redpanda debezium loadgen server grpc-proxy envoy web >> $LOG 2>&1
 else
     echo "$0: Internal error, aborting." 1>&2
     exit 1
 fi
 
 # fire and forget; will stop when compose stops.
 COMPOSE_LOG="${OUT_DIR}/docker-compose.log"
-(nohup docker-compose logs -f >& $COMPOSE_LOG < /dev/null &)
+(nohup docker compose logs -f >& $COMPOSE_LOG < /dev/null &)
 
 # avoid race with creation of log file above
 sleep 0.2

debezium/perf/stop_all.sh

@@ -2,4 +2,4 @@
 
 set -eu
 
-docker-compose stop && docker-compose down -v
+docker compose stop && docker compose down -v

docker-compose-common.yml

@@ -1,5 +1,3 @@
-version: "3.4"
-
 services:
   server:
     image: ${DEEPHAVEN_SERVER_IMAGE}
@@ -40,8 +38,8 @@ services:
           cpus: '4.0'
 
     # Allows the querying of this process jinfo/jmap
-    # docker-compose exec server jmap -heap 1
-    # docker-compose exec server jinfo 1
+    # docker compose exec server jmap -heap 1
+    # docker compose exec server jinfo 1
     #
     # Add NET_ADMIN to allow throttling network speeds
     # $ docker exec -it core_server_1 apt-get install iproute2

docker-compose.yml

@@ -9,8 +9,6 @@
 #
 # Please see README.md with more details on how to get started.
 
-version: "3.4"
-
 services:
   server:
     extends:

docker/server-slim/src/main/docker/Dockerfile

@@ -23,7 +23,7 @@ VOLUME /cache
 #     failure mode (via docker inspect, but only until that failure stales out).
 #   * If grpc_health_probe fails with any non-zero exit code, normalize to 1.
 # The net effect is that the server will have 3s until the earliest possible success, and
-# 33-36s until it is considered to have failed startup (so downstream docker-compose services
+# 33-36s until it is considered to have failed startup (so downstream docker compose services
 # will not attempt to start). If the server is running but trying to shut down, failure will
 # occur within about 9s, as the port will connect correctly, but respond that it isn't
 # available.

docker/web-plugin-packager/README.md

@@ -38,7 +38,7 @@ FROM ghcr.io/deephaven/server:latest
 RUN pip install deephaven-plugin-matplotlib
 ```
 
-After building, you need to specify using that build in your `docker-compose`. Do this by modifying the existing a `docker-compose.yml` file and replace the web and server definitions with the following:
+After building, you need to specify using that build in your `docker compose`. Do this by modifying the existing a `docker-compose.yml` file and replace the web and server definitions with the following:
 ```yaml
 services:
   server:
@@ -66,7 +66,7 @@ When you're done, your directory structure should look like:
     └── Dockerfile
 ```
 
-Everything's ready to go! Now you just need to run `docker-compose up` as normal, and you will be using your custom image with your JS plugins installed. See what you can create!
+Everything's ready to go! Now you just need to run `docker compose up` as normal, and you will be using your custom image with your JS plugins installed. See what you can create!
 
 ## JS Plugin Development
 

postgres/docker-compose.yml

@@ -1,5 +1,3 @@
-version: "3.4"
-
 services:
   postgres:
     build:

redpanda-apicurio/docker-compose.yml

@@ -1,8 +1,6 @@
 # Developer-oriented local docker compose file to run together
 # Deephaven Core and Redpanda (Kafka compatible event streaming)
 
-version: "3.4"
-
 services:
   server:
     extends:

redpanda-apicurio/examples/python/kafka-produce-avro.py

@@ -20,7 +20,7 @@
 #
 # == Common to all:
 #
-#  * Start the redpanda-apicurio compose: (cd redpanda-apicurio && docker-compose up --build)
+#  * Start the redpanda-apicurio compose: (cd redpanda-apicurio && docker compose up --build)
 #  * From web UI do:
 #
 #    > from deephaven import kafka_consumer as kc

redpanda-standalone/docker-compose.yml

@@ -1,4 +1,3 @@
-version: '3.4'
 services:
   redpanda:
     command:

redpanda/docker-compose.yml

@@ -1,8 +1,6 @@
 # Developer-oriented local docker compose file to run together
 # Deephaven Core and Redpanda (Kafka compatible event streaming)
 
-version: "3.4"
-
 services:
   server:
     extends:

redpanda/examples/python/kafka-produce-avro.py

@@ -24,7 +24,7 @@
 #
 # == Common to all:
 #
-#  * Start the redpanda compose: (cd redpanda && docker-compose up --build)
+#  * Start the redpanda compose: (cd redpanda && docker compose up --build)
 #  * From web UI do:
 #
 #    > from deephaven import kafka_consumer as kc

redpanda/examples/python/kafka-produce.py

@@ -23,7 +23,7 @@
 #
 # == Common to all:
 #
-#  * Start the redpanda compose: (cd redpanda && docker-compose up --build)
+#  * Start the redpanda compose: (cd redpanda && docker compose up --build)
 #  * From web UI do:
 #
 #    > from deephaven import kafka_consumer as kc