Syncing dev and main branches (#206)

This PR pulls the changes made in `dev` branch since last sync (May
2024) to the `main` branch.

---------

Co-authored-by: FarmVibes Release Pipeline <[email protected]>
Co-authored-by: Alex Crown <[email protected]>
Co-authored-by: Bruno Silva <[email protected]>
Co-authored-by: Eduardo Rodrigues <[email protected]>
Co-authored-by: Jessica Wolk <[email protected]>
Co-authored-by: Leonardo Nunes <[email protected]>
Co-authored-by: Naga Bilwanth Gangarapu <[email protected]>
Co-authored-by: Renato Luiz de Freitas Cunha <[email protected]>
Co-authored-by: Rick Gutierrez <[email protected]>
Co-authored-by: Roberto de Moura Estevão Filho <[email protected]>
Co-authored-by: Roberto Santos <[email protected]>
Co-authored-by: Sara Malvar <[email protected]>
Co-authored-by: Naga Bilwanth Gangarapu <[email protected]>

.devcontainer/devcontainer.json

@@ -0,0 +1,62 @@
+{
+    "build": {
+        "dockerfile": "../resources/docker/Dockerfile-devcontainer"
+    },
+    "overrideCommand": false,
+    "mounts": [
+        "type=bind,source=${localEnv:HOME}${localEnv:USERPROFILE},target=/user-home",
+        "source=terravibes-devcontainer-profile,target=/home/vscode,type=volume",
+        "target=/home/vscode/.vscode-server,type=volume",
+        "source=terravibes-devcontainer-var-lib-docker,target=/var/lib/docker,type=volume"
+    ],
+    "customizations": {
+        "vscode": {
+            "extensions": [
+                "charliermarsh.ruff",
+                "ms-python.python",
+                "ms-python.vscode-pylance",
+                "ms-toolsai.jupyter",
+                "ms-azuretools.vscode-docker",
+                "DavidAnson.vscode-markdownlint",
+                "ms-vscode-remote.remote-containers",
+                "eamodio.gitlens",
+                "mutantdino.resourcemonitor"
+            ],
+            "settings": {
+                "python.defaultInterpreterPath": "/opt/venv/bin/python",
+                "python.testing.pytestArgs": [
+                    "src",
+                    "ops"
+                ],
+                "python.testing.unittestEnabled": false,
+                "python.testing.pytestEnabled": true,
+                "editor.formatOnSave": true,
+                "editor.codeActionsOnSave": {
+                    "source.fixAll.ruff": "explicit",
+                    "source.organizeImports.ruff": "explicit"
+                },
+                "python.analysis.typeCheckingMode": "basic",
+                "python.analysis.diagnosticMode": "workspace",
+                "python.terminal.activateEnvironment": false,
+                "python.linting.flake8Enabled": true,
+                "git.autofetch": "all",
+                "terminal.integrated.allowChords": false,
+                "[python]": {
+                    "editor.formatOnSave": true,
+                    "editor.defaultFormatter": "charliermarsh.ruff"
+                }
+            }
+        }
+    },
+    "remoteUser": "vscode",
+    "updateRemoteUserUID": true,
+    "postCreateCommand": "sed 's/\r$//' .devcontainer/post-create.sh | bash -",
+    "runArgs": [
+        "--init",
+        "--privileged"
+    ],
+    "containerEnv": {
+        "USER": "vscode",
+        "PATH": "/home/vscode/.local/bin:/opt/venv/bin:/opt/conda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+    }
+}

.devcontainer/post-create.sh

@@ -0,0 +1,71 @@
+#!/bin/sh
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+
+DOCKER_VERSION=24.0.2
+VSCODE_HOME=/home/vscode
+
+if [ -d /user-home/.ssh ]; then
+    echo "Making user ssh available in container..."
+    mkdir -p $VSCODE_HOME/.ssh
+    chmod 0700 $VSCODE_HOME/.ssh
+    for f in /user-home/.ssh/*
+    do
+        cp "$f" $VSCODE_HOME/.ssh/"$(basename "$f")"
+        chmod 0600 $VSCODE_HOME/.ssh/"$(basename "$f")"
+    done
+fi
+
+# If the user has a git config file, copy it
+if [ -f /user-home/.gitconfig ]; then
+    echo "Copying user .gitconfig..."
+    cp /user-home/.gitconfig $VSCODE_HOME/.gitconfig
+    echo "Enabling HTTP use path, in case the user cloned with HTTP"
+    git config --global credential.useHttpPath true
+fi
+
+if [ "$(stat -c '%u' .)" != "$UID" ]; then
+    echo "The permissions of the current directory differ from the current user,"
+    echo "which means we're probably running in Docker under a Windows host..."
+    echo "Adding the current directory to the git safe directory list"
+    git config --global --add safe.directory /workspaces/TerraVibes
+fi
+
+sudo mkdir /opt/venv
+sudo chown vscode /opt/venv
+/opt/conda/bin/python3 -m venv --system-site-packages /opt/venv || exit 1
+/opt/venv/bin/pip install --upgrade pip
+
+if [[ "$(uname -a)" == *"WSL2"* ]]; then
+    # We're either in WSL2 or in a Windows host
+    echo "If we're on a Windows host, we need to convert files to unix mode..."
+    find cli scripts -type f -exec dos2unix --allow-chown {} \;
+fi
+
+sh -c "$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh)" "" --unattended
+zsh -c "zstyle ':omz:update' mode auto"
+zsh -c "zstyle ':omz:update' verbose minimal"
+git clone https://github.com/zsh-users/zsh-autosuggestions.git ~/.oh-my-zsh/plugins/zsh-autosuggestions
+git clone https://github.com/zsh-users/zsh-syntax-highlighting.git $ZSH_CUSTOM/plugins/zsh-syntax-highlighting
+git clone --depth 1 -- https://github.com/marlonrichert/zsh-autocomplete.git $ZSH_CUSTOM/plugins/zsh-autocomplete
+sed -i 's/plugins=(git)/plugins=(git zsh-autosuggestions zsh-syntax-highlighting zsh-autocomplete)/g' ~/.zshrc
+echo "export LD_LIBRARY_PATH=/opt/conda/lib:\$LD_LIBRARY_PATH" >> ~/.zshrc
+echo "export LD_LIBRARY_PATH=/opt/conda/lib:\$LD_LIBRARY_PATH" >> ~/.bashrc
+
+/opt/venv/bin/pip install --upgrade pyright
+/opt/venv/bin/pip install --upgrade "pytest" "anyio[trio]"
+sed -e '1,/dependencies:/d' < resources/envs/dev.yaml | \
+    sed 's/-//' | \
+    xargs /opt/venv/bin/pip install
+eval $(grep 'terravibes_packages=' < "scripts/setup_python_develop_env.sh")
+for package in $terravibes_packages
+do
+    /opt/venv/bin/pip install -e src/$package
+done
+
+sudo mkdir -p /opt/terravibes/ops
+sudo ln -sf $(pwd)/op_resources /opt/terravibes/ops/resources
+sudo mkdir /app
+sudo ln -sf $(pwd)/ops /app/ops
+sudo ln -sf $(pwd)/workflows /app/workflows

.gitattributes

@@ -0,0 +1,4 @@
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.xls filter=lfs diff=lfs merge=lfs -text
+*.kml filter=lfs diff=lfs merge=lfs -text
+*.tif filter=lfs diff=lfs merge=lfs -text

.github/workflows/base-build.yml

@@ -0,0 +1,64 @@
+name: Build base images
+on:
+  workflow_dispatch:
+
+permissions:
+  id-token: write
+  contents: write
+
+jobs:
+
+  build-and-push:
+    runs-on: ubuntu-latest
+    environment: build
+    strategy:
+      matrix:
+        include:
+        - dockerfile: Dockerfile-services-base
+          image-name: services-base
+        - dockerfile: Dockerfile-worker-base
+          image-name: worker-base
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: dev
+      - name: 'Az CLI login'
+        uses: azure/login@v1
+        with:
+          client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+      - name: 'Build and push image'
+        run: |
+          az acr login -n ${{ secrets.ACR_NAME }}
+          VERSION_TAG=${{ github.run_id }}
+          export IMAGE_TAG=${{ secrets.ACR_NAME }}.azurecr.io/unlisted/farmai/terravibes/${{ matrix.image-name }}:$VERSION_TAG
+          docker build . -f ./resources/docker/${{ matrix.dockerfile }} -t $IMAGE_TAG
+          docker push $IMAGE_TAG
+  update-tags:
+    needs: build-and-push 
+    runs-on: ubuntu-latest
+    environment: build
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: dev
+          ssh-key: ${{ secrets.WORKFLOW_KEY }}
+      - name: 'Update tags' 
+        run: |
+          TAG=${{ github.run_id }}
+          sed -i "s|\(\s\+image.*:\).*|\1${TAG}|" ./.github/workflows/lint-test.yml
+          sed -i "s|\(FROM.*:\).*|\1${TAG}|" ./resources/docker/Dockerfile-api_orchestrator
+          sed -i "s|\(FROM.*:\).*|\1${TAG}|" ./resources/docker/Dockerfile-cache
+          sed -i "s|\(FROM.*:\).*|\1${TAG}|" ./resources/docker/Dockerfile-worker
+          sed -i "s|\(FROM.*:\).*|\1${TAG}|" ./resources/docker/Dockerfile-devcontainer
+      - name: Commit changes
+        run: |
+          git config --global user.email "[email protected]"
+          git config --global user.name "FarmVibes.AI Release Pipeline"
+          BRANCH=update-base-${{ github.run_id }}
+          git checkout -b $BRANCH
+          git add ./.github/workflows/lint-test.yml
+          git add ./resources/docker/
+          git commit -m "Update base tag to latest image"
+          git push --set-upstream origin $BRANCH

.github/workflows/docker-build.yml

@@ -0,0 +1,39 @@
+name: Build service images
+on:
+  push:
+    branches:
+      - dev
+      - main
+
+permissions:
+  id-token: write
+  contents: read
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    environment: build
+    strategy:
+      matrix:
+        include:
+        - dockerfile: Dockerfile-api_orchestrator
+          image-name: api-orchestrator
+        - dockerfile: Dockerfile-worker
+          image-name: worker
+        - dockerfile: Dockerfile-cache
+          image-name: cache
+    steps:
+      - uses: actions/checkout@v4
+      - name: 'Az CLI login'
+        uses: azure/login@v1
+        with:
+          client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+      - name: 'Build and push image'
+        run: |
+          az acr login -n ${{ secrets.ACR_NAME }}
+          export VERSION_TAG=${GITHUB_REF#refs/heads/}
+          export IMAGE_TAG=${{ secrets.ACR_NAME }}.azurecr.io/unlisted/farmai/terravibes/${{ matrix.image-name }}:$VERSION_TAG
+          docker build . -f ./resources/docker/${{ matrix.dockerfile }} -t $IMAGE_TAG
+          docker push $IMAGE_TAG
+