Refactor docker images in EVerest

* Add devcontainer template
* Add easy-setup script `setup-devcontainer.sh`
* Move `mosquitto` image from `everest-utils` repository
* Move `steve` image from `everest-utils` repository
* Add nodered image
* Update README.md
* Add `everest_dev_tool`

Signed-off-by: Andreas Heinrich <[email protected]>

.github/workflows/deploy-docker-images.yml

@@ -0,0 +1,100 @@
+name: Build and push docker images
+
+on:
+  pull_request: {}
+  push:
+    branches:
+      - '**'
+    tags:
+      - 'v*'
+
+env:
+  REGISTRY: ghcr.io
+  DOCKER_DIRECTORY: docker/images
+  PLATFORMS: |
+    linux/amd64
+  PATH_TO_DEPLOY_DOCKER_IMAGES_WORKFLOW: .github/workflows/deploy-docker-images.yml
+
+jobs:
+  env-setup:
+    # Since env variables can't be passed to reusable workflows, we need to pass them as outputs
+    name: Setup environment
+    runs-on: ubuntu-22.04
+    outputs:
+      docker_registry: ${{ env.REGISTRY }}
+      docker_directory: ${{ env.DOCKER_DIRECTORY }}
+      platforms: ${{ env.PLATFORMS }}
+      path_to_deploy_docker_images_workflow: ${{ env.PATH_TO_DEPLOY_DOCKER_IMAGES_WORKFLOW }}
+    steps:
+      - id: check
+        run: |
+          echo "Setting up environment"
+  mosquitto:
+    needs:
+      - env-setup
+    name: Build and push mosquitto docker image
+    uses: everest/everest-ci/.github/workflows/[email protected]
+    secrets:
+      SA_GITHUB_PAT: ${{ secrets.SA_GITHUB_PAT }}
+      SA_GITHUB_USERNAME: ${{ secrets.SA_GITHUB_USERNAME }}
+    with:
+      image_name: ${{ github.event.repository.name }}/mosquitto
+      directory: ${{ needs.env-setup.outputs.docker_directory }}/mosquitto
+      docker_registry: ${{ needs.env-setup.outputs.docker_registry }}
+      github_ref_before: ${{ github.event.before }}
+      github_ref_after: ${{ github.event.after }}
+      platforms: ${{ needs.env-setup.outputs.platforms }}
+      depends_on_paths: |
+        ${{ needs.env-setup.outputs.path_to_deploy_docker_images_workflow }}
+  steve:
+    needs:
+      - env-setup
+    name: Build and push steve docker image
+    uses: everest/everest-ci/.github/workflows/[email protected]
+    secrets:
+      SA_GITHUB_PAT: ${{ secrets.SA_GITHUB_PAT }}
+      SA_GITHUB_USERNAME: ${{ secrets.SA_GITHUB_USERNAME }}
+    with:
+      image_name: ${{ github.event.repository.name }}/steve
+      directory: ${{ needs.env-setup.outputs.docker_directory }}/steve
+      docker_registry: ${{ needs.env-setup.outputs.docker_registry }}
+      github_ref_before: ${{ github.event.before }}
+      github_ref_after: ${{ github.event.after }}
+      platforms: ${{ needs.env-setup.outputs.platforms }}
+      depends_on_paths: |
+        ${{ needs.env-setup.outputs.path_to_deploy_docker_images_workflow }}
+  mqtt-explorer:
+    needs:
+      - env-setup
+    name: Build and push mqtt-explorer docker image
+    uses: everest/everest-ci/.github/workflows/[email protected]
+    secrets:
+      SA_GITHUB_PAT: ${{ secrets.SA_GITHUB_PAT }}
+      SA_GITHUB_USERNAME: ${{ secrets.SA_GITHUB_USERNAME }}
+    with:
+      image_name: ${{ github.event.repository.name }}/mqtt-explorer
+      directory: ${{ needs.env-setup.outputs.docker_directory }}/mqtt-explorer
+      docker_registry: ${{ needs.env-setup.outputs.docker_registry }}
+      github_ref_before: ${{ github.event.before }}
+      github_ref_after: ${{ github.event.after }}
+      platforms: ${{ needs.env-setup.outputs.platforms }}
+      depends_on_paths: |
+        ${{ needs.env-setup.outputs.path_to_deploy_docker_images_workflow }}
+  nodered:
+    needs:
+      - env-setup
+    name: Build and push nodered docker image
+    uses: everest/everest-ci/.github/workflows/[email protected]
+    secrets:
+      SA_GITHUB_PAT: ${{ secrets.SA_GITHUB_PAT }}
+      SA_GITHUB_USERNAME: ${{ secrets.SA_GITHUB_USERNAME }}
+    with:
+      image_name: ${{ github.event.repository.name }}/nodered
+      directory: ${{ needs.env-setup.outputs.docker_directory }}/nodered
+      docker_registry: ${{ needs.env-setup.outputs.docker_registry }}
+      github_ref_before: ${{ github.event.before }}
+      github_ref_after: ${{ github.event.after }}
+      platforms: ${{ needs.env-setup.outputs.platforms }}
+      depends_on_paths: |
+        ${{ needs.env-setup.outputs.path_to_deploy_docker_images_workflow }}
+    

README.md

@@ -5,3 +5,46 @@ This subproject contains all utility files for setting up your development envir
 You can install [edm](dependency_manager/README.md) very easy using pip.
 
 All documentation and the issue tracking can be found in our main repository here: https://github.com/EVerest/everest
+
+## Easy Dev Environment Setup
+
+To setup a devcontainer in your workspace you can use the following command to run the `setup_devcontainer.sh` script locally.
+
+### 1. Prerequisites
+
+Create a new directory and navigate into it. This directory will be your new workspace or use an existing one.
+
+```bash
+mkdir my-workspace
+cd my-workspace
+```
+
+### 2. Run the setup script
+
+Run the following command to setup the devcontainer.
+
+```bash
+export BRANCH="main" && bash -c "$(curl -s --variable %BRANCH=main --expand-url https://raw.githubusercontent.com/EVerest/everest-dev-environment/{{BRANCH}}/devcontainer/setup-devcontainer.sh)"
+```
+
+The script will ask you for the following information:
+1. Workspace directory: Default is the current directory. You can keep the default by pressing enter.
+2. everest-dev-environment version: Default is 'main'. You can keep the default by pressing enter.
+
+### 3. Open in VS Code
+
+After the script has finished, open the workspace in Visual Studio Code.
+
+```bash
+code .
+```
+
+VS Code will ask you to reopen the workspace in a container. Click on the button `Reopen in Container`.
+
+### 4. Getting started
+
+As your set up dev environment suggests when you open a terminal, you can setup your EVerest workspace by running the following command:
+
+```bash
+everest clone everest-core
+```

devcontainer/setup-devcontainer.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+set -e
+
+read -p "Enter the workspace directory (default is the current directory): " WORKSPACE_DIR
+if [ -z "$WORKSPACE_DIR" ]; then
+    WORKSPACE_DIR="./"
+fi
+WORKSPACE_DIR=$(realpath -m "$WORKSPACE_DIR")
+
+read -p "Enter the version of the everest-dev-environment (default is 'main'): " VERSION
+if [ -z "$VERSION" ]; then
+    VERSION="main"
+fi
+
+echo "Create the workspace directory '$WORKSPACE_DIR' if it does not exist"
+mkdir -p $WORKSPACE_DIR
+
+if [ "$(ls -A $WORKSPACE_DIR)" ]; then
+    # The workspace directory is not empty, warning do you want to continue?
+    read -p "The workspace directory is not empty, do you want to continue? (y/N): " -r
+    if [[ $REPLY =~ ^[Nn]$ || $REPLY = "" ]]; then
+        echo "Exiting.."
+        exit 1
+    elif [[ ! $REPLY =~ ^[Yy]$ ]]; then
+        echo "Invalid input. Exiting.."
+        exit 1
+    fi
+fi
+
+TMP_DIR="$WORKSPACE_DIR/tmp-everest-dev-environment"
+echo "Clone the everest-dev-environment repository to the workspace directory with the version $VERSION, temporarily.."
+git clone --quiet --depth 1 --single-branch --branch "$VERSION" https://github.com/EVerest/everest-dev-environment.git "$TMP_DIR"
+
+echo "Copy the template devcontainer configuration files to the workspace directory"
+cp -n -r $TMP_DIR/devcontainer/template/. $WORKSPACE_DIR/
+
+echo "Remove the everest-dev-environment repository"
+rm -rf "$TMP_DIR"

devcontainer/template/.devcontainer/docker-compose.yml

@@ -0,0 +1,40 @@
+volumes:
+  ocpp-db-data:
+    external: false
+  node-red-data:
+    external: false
+
+services:
+  mqtt-server:
+    image: ghcr.io/everest/everest-dev-environment/mosquitto:v0.7.0
+    ports:
+      # allow multiple ports for host to avoid conflicts with other dev environments
+      - 1883-1983:1883
+      - 9001-9101:9001
+
+  ocpp-db:
+    image: mariadb:10.4.30 # pinned to patch-version because https://github.com/steve-community/steve/pull/1213
+    volumes:
+      - ocpp-db-data:/var/lib/mysql
+    ports:
+      # allow multiple ports for host to avoid conflicts with other dev environments
+      - 13306-13406:3306
+    environment:
+      MYSQL_RANDOM_ROOT_PASSWORD: "yes"
+      MYSQL_DATABASE: ocpp-db
+      MYSQL_USER: ocpp
+      MYSQL_PASSWORD: ocpp
+  steve:
+    image: ghcr.io/everest/everest-dev-environment/steve:v0.7.0
+    ports:
+      # allow multiple ports for host to avoid conflicts with other dev environments
+      - 8180-8280:8180
+      - 8443-8543:8443
+    depends_on:
+      - ocpp-db
+  mqtt-explorer:
+    image: ghcr.io/everest/everest-dev-environment/mqtt-explorer:v0.7.0
+    depends_on:
+      - mqtt-server
+    ports:
+      - 4000-4100:4000

devcontainer/template/.devcontainer/general-devcontainer/Dockerfile

@@ -0,0 +1,17 @@
+# syntax=docker/dockerfile:1
+FROM ghcr.io/everest/everest-ci/dev-env-base:v1.3.2
+
+# Update the package list
+RUN sudo apt update
+
+# EVerest Development Tool - Dependencies
+RUN pip install --break-system-packages \
+    docker==7.1.0
+# EVerest Development Tool
+ARG DEV_ENV_TOOL_VERSION=v0.7.0
+RUN python3 -m pip install --break-system-packages \
+    git+https://github.com/EVerest/everest-dev-environment@${DEV_ENV_TOOL_VERSION}#subdirectory=everest_dev_tool
+
+RUN echo "echo \"🏔️ 🚘 Welcome to the EVerest development environment!\"" >> ${HOME}/.bashrc
+RUN echo "echo \"To initialize the EVerest core repository everest-core in your workspace please run the following command:\"" >> ${HOME}/.bashrc
+RUN echo "echo \"everest clone everest-core\"" >> ${HOME}/.bashrc

devcontainer/template/.devcontainer/general-devcontainer/devcontainer.json

@@ -0,0 +1,63 @@
+{
+  "name": "EVerest - ${localWorkspaceFolderBasename}",
+  "dockerComposeFile": ["../docker-compose.yml", "./docker-compose.devcontainer.yml"],
+  "service": "devcontainer",
+  "runServices": ["devcontainer"],
+  "remoteUser": "docker",
+  "workspaceFolder": "/workspace",
+  "forwardPorts": [
+    "mqtt-explorer:4000",
+    "steve:8180"
+  ],
+  "portsAttributes": {
+    "mqtt-explorer:4000": {
+      "label": "MQTT Explorer - WebView"
+    },
+    "steve:8180": {
+      "label": "Steve - WebTool"
+    }
+  },
+  "otherPortsAttributes": {
+    "onAutoForward": "notify",
+    "protocol": "http",
+    "requireLocalPort": false
+  },
+  "customizations": {
+    "vscode": {
+      "settings": {
+        "terminal.integrated.profiles.linux": {
+          "bash": {
+            "path": "/bin/bash",
+            "icon": "terminal-bash",
+            "args": ["-l"]
+          }
+        },
+        "terminal.integrated.defaultProfile.linux": "bash",
+        "python.pythonPath": "/usr/bin/python3",
+        "python.defaultInterpreterPath": "/usr/bin/python3",
+        "editor.rulers": [79, 120],
+        // RST/Sphinx language server
+        "esbonio.sphinx.buildDir": "${workspaceFolder}/everest/docs/_build",
+        "esbonio.sphinx.confDir": "${workspaceFolder}/everest/docs",
+        // RST
+        "restructuredtext.preview.scrollEditorWithPreview": false,
+        "restructuredtext.pythonRecommendation.disabled": true,
+        "liveServer.settings.root": "/everest/docs/_build/html"
+      },
+      "extensions": [
+        // language support CPP
+        "ms-vscode.cpptools",
+        // language support cmake
+        "twxs.cmake",
+        "ms-vscode.cmake-tools",
+				// language support python
+				"ms-python.python",
+				// language support restructured text
+				"lextudio.restructuredtext",
+				"trond-snekvik.simple-rst",
+				// doc live server
+				"ritwickdey.liveserver"
+      ]
+    }
+  }
+}

devcontainer/template/.devcontainer/general-devcontainer/docker-compose.devcontainer.yml

@@ -0,0 +1,48 @@
+networks:
+  docker-proxy-network:
+    internal: true
+
+volumes:
+  cpm-source-cache:
+    name: everest-cpm-source-cache
+
+services:
+  docker-proxy:
+    image: tecnativa/docker-socket-proxy:latest
+    volumes:
+      - type: bind
+        source: /var/run/docker.sock
+        target: /var/run/docker.sock
+    environment:
+      - CONTAINERS=1
+      - IMAGES=1
+      - POST=1
+      - NETWORKS=1
+      - VOLUMES=1
+    networks:
+      - docker-proxy-network
+
+  devcontainer:
+    depends_on:
+      - docker-proxy
+    build:
+      context: ./general-devcontainer
+      dockerfile: Dockerfile
+    volumes:
+      - type: bind
+        source: ..
+        target: /workspace
+      - type: volume
+        source: cpm-source-cache
+        target: /home/docker/.cache/cpm
+    command: sleep infinity
+    environment:
+      MQTT_SERVER_ADDRESS: mqtt-server
+      MQTT_SERVER_PORT: 1883
+      DOCKER_HOST: tcp://docker-proxy:2375
+      CPM_SOURCE_CACHE: /home/docker/.cache/cpm
+    networks:
+      - docker-proxy-network
+      - default
+    sysctls:
+      - net.ipv6.conf.all.disable_ipv6=0

docker/images/mosquitto/Dockerfile

@@ -0,0 +1,3 @@
+FROM eclipse-mosquitto:2.0.10
+
+COPY mosquitto.conf /mosquitto/config/mosquitto.conf