Add components needed to build docker images

Co-authored-by: Alex Crown <[email protected]> Co-authored-by: Bruno Silva <[email protected]> Co-authored-by: Eduardo Rodrigues <[email protected]> Co-authored-by: Jessica Wolk <[email protected]> Co-authored-by: Leonardo Nunes <[email protected]> Co-authored-by: Naga Bilwanth Gangarapu <[email protected]> Co-authored-by: Rafael Padilha <[email protected]> Co-authored-by: Renato Luiz de Freitas Cunha <[email protected]> Co-authored-by: Rick Gutierrez <[email protected]> Co-authored-by: Roberto de Moura Estevão Filho <[email protected]> Co-authored-by: Roberto Santos <[email protected]> Co-authored-by: Sara Malvar <[email protected]>
microsoft · Jul 6, 2024 · ca4b204 · ca4b204
1 parent 37958a2
commit ca4b204
Show file tree

Hide file tree

Showing 673 changed files with 51,199 additions and 71 deletions.
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -0,0 +1,62 @@
+{
+    "build": {
+        "dockerfile": "../resources/docker/Dockerfile-devcontainer"
+    },
+    "overrideCommand": false,
+    "mounts": [
+        "type=bind,source=${localEnv:HOME}${localEnv:USERPROFILE},target=/user-home",
+        "source=terravibes-devcontainer-profile,target=/home/vscode,type=volume",
+        "target=/home/vscode/.vscode-server,type=volume",
+        "source=terravibes-devcontainer-var-lib-docker,target=/var/lib/docker,type=volume"
+    ],
+    "customizations": {
+        "vscode": {
+            "extensions": [
+                "charliermarsh.ruff",
+                "ms-python.python",
+                "ms-python.vscode-pylance",
+                "ms-toolsai.jupyter",
+                "ms-azuretools.vscode-docker",
+                "DavidAnson.vscode-markdownlint",
+                "ms-vscode-remote.remote-containers",
+                "eamodio.gitlens",
+                "mutantdino.resourcemonitor"
+            ],
+            "settings": {
+                "python.defaultInterpreterPath": "/opt/venv/bin/python",
+                "python.testing.pytestArgs": [
+                    "src",
+                    "ops"
+                ],
+                "python.testing.unittestEnabled": false,
+                "python.testing.pytestEnabled": true,
+                "editor.formatOnSave": true,
+                "editor.codeActionsOnSave": {
+                    "source.fixAll.ruff": "explicit",
+                    "source.organizeImports.ruff": "explicit"
+                },
+                "python.analysis.typeCheckingMode": "basic",
+                "python.analysis.diagnosticMode": "workspace",
+                "python.terminal.activateEnvironment": false,
+                "python.linting.flake8Enabled": true,
+                "git.autofetch": "all",
+                "terminal.integrated.allowChords": false,
+                "[python]": {
+                    "editor.formatOnSave": true,
+                    "editor.defaultFormatter": "charliermarsh.ruff"
+                }
+            }
+        }
+    },
+    "remoteUser": "vscode",
+    "updateRemoteUserUID": true,
+    "postCreateCommand": "sed 's/\r$//' .devcontainer/post-create.sh | bash -",
+    "runArgs": [
+        "--init",
+        "--privileged"
+    ],
+    "containerEnv": {
+        "USER": "vscode",
+        "PATH": "/home/vscode/.local/bin:/opt/venv/bin:/opt/conda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+    }
+}
diff --git a/.devcontainer/post-create.sh b/.devcontainer/post-create.sh
@@ -0,0 +1,68 @@
+#!/bin/sh
+
+DOCKER_VERSION=24.0.2
+VSCODE_HOME=/home/vscode
+
+if [ -d /user-home/.ssh ]; then
+    echo "Making user ssh available in container..."
+    mkdir -p $VSCODE_HOME/.ssh
+    chmod 0700 $VSCODE_HOME/.ssh
+    for f in /user-home/.ssh/*
+    do
+        cp "$f" $VSCODE_HOME/.ssh/"$(basename "$f")"
+        chmod 0600 $VSCODE_HOME/.ssh/"$(basename "$f")"
+    done
+fi
+
+# If the user has a git config file, copy it
+if [ -f /user-home/.gitconfig ]; then
+    echo "Copying user .gitconfig..."
+    cp /user-home/.gitconfig $VSCODE_HOME/.gitconfig
+    echo "Enabling HTTP use path, in case the user cloned with HTTP"
+    git config --global credential.useHttpPath true
+fi
+
+if [ "$(stat -c '%u' .)" != "$UID" ]; then
+    echo "The permissions of the current directory differ from the current user,"
+    echo "which means we're probably running in Docker under a Windows host..."
+    echo "Adding the current directory to the git safe directory list"
+    git config --global --add safe.directory /workspaces/TerraVibes
+fi
+
+sudo mkdir /opt/venv
+sudo chown vscode /opt/venv
+/opt/conda/bin/python3 -m venv --system-site-packages /opt/venv || exit 1
+/opt/venv/bin/pip install --upgrade pip
+
+if [[ "$(uname -a)" == *"WSL2"* ]]; then
+    # We're either in WSL2 or in a Windows host
+    echo "If we're on a Windows host, we need to convert files to unix mode..."
+    find cli scripts -type f -exec dos2unix --allow-chown {} \;
+fi
+
+sh -c "$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh)" "" --unattended
+zsh -c "zstyle ':omz:update' mode auto"
+zsh -c "zstyle ':omz:update' verbose minimal"
+git clone https://github.com/zsh-users/zsh-autosuggestions.git ~/.oh-my-zsh/plugins/zsh-autosuggestions
+git clone https://github.com/zsh-users/zsh-syntax-highlighting.git $ZSH_CUSTOM/plugins/zsh-syntax-highlighting
+git clone --depth 1 -- https://github.com/marlonrichert/zsh-autocomplete.git $ZSH_CUSTOM/plugins/zsh-autocomplete
+sed -i 's/plugins=(git)/plugins=(git zsh-autosuggestions zsh-syntax-highlighting zsh-autocomplete)/g' ~/.zshrc
+echo "export LD_LIBRARY_PATH=/opt/conda/lib:\$LD_LIBRARY_PATH" >> ~/.zshrc
+echo "export LD_LIBRARY_PATH=/opt/conda/lib:\$LD_LIBRARY_PATH" >> ~/.bashrc
+
+/opt/venv/bin/pip install --upgrade pyright
+/opt/venv/bin/pip install --upgrade "pytest" "anyio[trio]"
+sed -e '1,/dependencies:/d' < resources/envs/dev.yaml | \
+    sed 's/-//' | \
+    xargs /opt/venv/bin/pip install
+eval $(grep 'terravibes_packages=' < "scripts/setup_python_develop_env.sh")
+for package in $terravibes_packages
+do
+    /opt/venv/bin/pip install -e src/$package
+done
+
+sudo mkdir -p /opt/terravibes/ops
+sudo ln -sf $(pwd)/op_resources /opt/terravibes/ops/resources
+sudo mkdir /app
+sudo ln -sf $(pwd)/ops /app/ops
+sudo ln -sf $(pwd)/workflows /app/workflows
diff --git a/.gitattributes b/.gitattributes
@@ -0,0 +1,4 @@
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.xls filter=lfs diff=lfs merge=lfs -text
+*.kml filter=lfs diff=lfs merge=lfs -text
+*.tif filter=lfs diff=lfs merge=lfs -text
diff --git a/.gitignore b/.gitignore
@@ -122,12 +122,9 @@ resources/test/
 
 !terravibes/vibe/lib/
 .cspell/*
-user-interface/package-lock.json
-carbon/notebook/farmbeats/*
-carbon/notebook/data/*
 .env.development
 test-output.xml
 /outputs/*
-/op_resources/*
+venv/
 
 makeenv
diff --git a/.ruff.toml b/.ruff.toml
@@ -0,0 +1,17 @@
+src= ["src"]
+line-length = 100
+extend-include = ["*.ipynb"]
+
+[lint]
+select = ["E", "F", "I", "W"]
+ignore = ["E203"]
+exclude = [".git", "__pycache__"]
+
+[lint.pycodestyle]
+max-line-length = 100
+
+[lint.per-file-ignores]
+"__init__.py" = ["F401"]
+
+[format]
+quote-style = "double"
diff --git a/Makefile b/Makefile
@@ -0,0 +1,199 @@
+.PHONY: help local clean revert-% revert clean-% local-% cluster set-image %-base
+
+SHELL = /bin/bash
+
+export PATH := $(HOME)/.config/farmvibes-ai:$(PATH)
+
+CACHE_DEPLOYMENT := terravibes-cache
+ORCHESTRATOR_DEPLOYMENT := terravibes-orchestrator
+REST_API_DEPLOYMENT := terravibes-rest-api
+DATA_OPS_DEPLOYMENT := terravibes-data-ops
+WORKER_DEPLOYMENT := terravibes-worker
+
+CACHE_REPO := farmai/terravibes/cache
+ORCHESTRATOR_REPO := farmai/terravibes/api-orchestrator
+REST_API_REPO := farmai/terravibes/api-orchestrator
+DATA_OPS_REPO := farmai/terravibes/cache
+WORKER_REPO := farmai/terravibes/worker
+
+CONTAINER_DEBUG_PORT := 5678
+REST_API_DEBUG_PORT := 5678
+ORCHESTRATOR_DEBUG_PORT := 5679
+CACHE_DEBUG_PORT := 5680
+WORKER_DEBUG_PORT := 5681
+DATA_OPS_DEBUG_PORT := 5682
+
+CURRENT_CACHE_REPLICAS := $(shell env PATH=$(PATH) kubectl get deployment $(CACHE_DEPLOYMENT) -o jsonpath='{.status.replicas}')
+CURRENT_REST_API_REPLICAS := $(shell env PATH=$(PATH) kubectl get deployment $(REST_API_DEPLOYMENT) -o jsonpath='{.status.replicas}')
+CURRENT_ORCHESTRATOR_REPLICAS := $(shell env PATH=$(PATH) kubectl get deployment $(ORCHESTRATOR_DEPLOYMENT) -o jsonpath='{.status.replicas}')
+CURRENT_DATA_OPS_REPLICAS := $(shell env PATH=$(PATH) kubectl get deployment $(DATA_OPS_DEPLOYMENT) -o jsonpath='{.status.replicas}')
+CURRENT_WORKER_REPLICAS := $(shell env PATH=$(PATH) kubectl get deployment $(WORKER_DEPLOYMENT) -o jsonpath='{.status.replicas}')
+
+TAG := tmp-$(shell date +%s)
+ROOT := $(shell git rev-parse --show-toplevel)
+
+build_cluster := env FARMVIBES_AI_IMAGE_PREFIX=terravibes- CONTAINER_REGISTRY_BASE=mcr.microsoft.com bash farmvibes-ai local setup
+base_image_name := grep -oE 'FROM ([-a-zA-Z0-9@:%._\+~\#=]{2,256}\.[a-z]{2,6}\b([-a-zA-Z0-9@:%_\+.~\#?&//=]*))' FILE | cut -d ' ' -f 2
+
+define transform_image_name
+$(shell docker ps | grep registry | rev | cut -d ' ' -f 1 | rev):5000/$(1)
+endef
+
+help: ## Shows this help message
+	@echo -e This is the farmvibes.ai makefile. Supported targets are:\\n
+	@grep -E -h '\s##\s' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
+
+local: cluster local-rest-api local-cache local-worker local-orchestrator local-data-ops ## Builds all images locally and deploys them into the local farmvibes.ai cluster
+	[ -z $(WAIT_AT_THE_END) ] || kubectl delete pods -l backend=terravibes && \
+		kubectl wait --for=condition=Available deployment --timeout=300s -l backend=terravibes
+
+revert: cluster revert-rest-api revert-cache revert-worker revert-orchestrator ## Reverts all images to the official version
+
+services-base: resources/docker/Dockerfile-services-base
+	@docker manifest inspect `$(subst FILE,$<,$(base_image_name))` || \
+		az acr login -n `$(subst FILE,$<,$(base_image_name)) | cut -d / -f 1 | sed 's|.azurecr.io||g'` || \
+		echo "Failed to log into container registry. Please perform an `az login` and try again"
+
+%-base: resources/docker/Dockerfile-%
+	@docker manifest inspect `$(subst FILE,$<,$(base_image_name))` || \
+		az acr login -n `$(subst FILE,$<,$(base_image_name)) | cut -d / -f 1 | sed 's|.azurecr.io||g'` || \
+		echo "Failed to log into container registry. Please perform an `az login` and try again"
+
+delete-%:
+	kubectl scale deployment $(subst delete-,,$@) --replicas=0
+	kubectl delete pod --wait=true -l app=$(subst delete-,,$@) --grace-period=0 --force
+	kubectl rollout status deployment $(subst delete-,,$@)
+
+repo-%:
+	docker pull $(CONTAINER_REGISTRY_BASE)/$(subst repo-,,$@):$(FARMVIBES_AI_IMAGE_TAG)
+
+set-image:
+	kubectl set image deployment $(DEPLOYMENT) "*=$(IMAGE_FULL_REFERENCE)"
+	kubectl rollout status deployment $(DEPLOYMENT)
+
+set-registry-image: push-image
+	DEPLOYMENT=$(DEPLOYMENT) IMAGE_FULL_REFERENCE=$(call transform_image_name,$(IMAGE_FULL_REFERENCE)) make -C . set-image
+
+push-image:
+	docker tag $(IMAGE_FULL_REFERENCE) 127.0.0.1:5000/$(IMAGE_FULL_REFERENCE)
+	docker push 127.0.0.1:5000/$(IMAGE_FULL_REFERENCE)
+
+scale:
+	kubectl scale deployment $(DEPLOYMENT) --replicas=$(shell [ "$(REPLICAS)" ] && echo "$(REPLICAS)" || echo 1)
+	[ ! -z $(WAIT_AT_THE_END) ] || kubectl wait --for=condition=Available deployment --timeout=300s $(DEPLOYMENT)
+
+# Have to replace Xfrozen_modules=on with Xfrozen_modules=off in the deployment
+disable-frozen-modules:
+	kubectl get deployment $(DEPLOYMENT) -o yaml | sed 's|Xfrozen_modules=on|Xfrozen_modules=off|g' | kubectl apply -f -
+
+add-debug-flag:
+	kubectl get deployment $(DEPLOYMENT) -o yaml | sed 's|\(\s\+-\)\(.*port=3000\)|\1\2\n\1 --debug|' | kubectl apply -f -
+
+add-debug-flag-agent:
+	kubectl get deployment $(DEPLOYMENT) -o yaml | sed 's|\(\s\+-\)\(.*port=3000\)|\1\2\n\1 debug.activate=true|' | kubectl apply -f -
+
+local-rest-api: cluster local-rest-api-orchestrator delete-$(REST_API_DEPLOYMENT) ## Builds and deploys a local REST API image (enabling debug)
+	DEPLOYMENT=$(REST_API_DEPLOYMENT) IMAGE_FULL_REFERENCE=$(REST_API_REPO):$(TAG) $(MAKE) -C . set-registry-image
+	@kubectl get deployment $(REST_API_DEPLOYMENT) -o json | grep -v last | grep -qo -- --debug || DEPLOYMENT=$(REST_API_DEPLOYMENT) $(MAKE) -C . add-debug-flag
+	DEPLOYMENT=$(REST_API_DEPLOYMENT) $(MAKE) -C . disable-frozen-modules
+	DEPLOYMENT=$(REST_API_DEPLOYMENT) REPLICAS=$(CURRENT_REST_API_REPLICAS) $(MAKE) scale
+
+revert-rest-api: cluster repo-$(REST_API_REPO) delete-$(REST_API_DEPLOYMENT) ## Reverts the REST API deployment to use the official image
+	DEPLOYMENT=$(REST_API_DEPLOYMENT) IMAGE_FULL_REFERENCE=$(CONTAINER_REGISTRY_BASE)/$(REST_API_REPO):$(FARMVIBES_AI_IMAGE_TAG) $(MAKE) set-registry-image
+	DEPLOYMENT=$(REST_API_DEPLOYMENT) REPLICAS=$(CURRENT_REST_API_REPLICAS) make scale
+
+local-orchestrator: cluster local-rest-api-orchestrator delete-$(ORCHESTRATOR_DEPLOYMENT) ## Builds and deploys a local ORCHESTRATOR image (enabling debug)
+	DEPLOYMENT=$(ORCHESTRATOR_DEPLOYMENT) IMAGE_FULL_REFERENCE=$(ORCHESTRATOR_REPO):$(TAG) $(MAKE) -C . set-registry-image
+	@kubectl get deployment $(ORCHESTRATOR_DEPLOYMENT) -o json | grep -v last | grep -qo -- --debug || DEPLOYMENT=$(ORCHESTRATOR_DEPLOYMENT) $(MAKE) -C . add-debug-flag
+	DEPLOYMENT=$(ORCHESTRATOR_DEPLOYMENT) $(MAKE) -C . disable-frozen-modules
+	DEPLOYMENT=$(ORCHESTRATOR_DEPLOYMENT) REPLICAS=$(CURRENT_ORCHESTRATOR_REPLICAS) $(MAKE) scale
+
+revert-orchestrator: cluster repo-$(ORCHESTRATOR_REPO) delete-$(ORCHESTRATOR_DEPLOYMENT) ## Reverts the ORCHESTRATOR deployment to use the official image
+	DEPLOYMENT=$(ORCHESTRATOR_DEPLOYMENT) IMAGE_FULL_REFERENCE=$(CONTAINER_REGISTRY_BASE)/$(ORCHESTRATOR_REPO):$(FARMVIBES_AI_IMAGE_TAG) $(MAKE) set-registry-image
+	DEPLOYMENT=$(ORCHESTRATOR_DEPLOYMENT) REPLICAS=$(CURRENT_ORCHESTRATOR_REPLICAS) make scale
+
+local-data-ops: cluster local-cache-repo delete-$(DATA_OPS_DEPLOYMENT) ## Builds and deploys a local data ops image (enabling debug)
+	DEPLOYMENT=$(DATA_OPS_DEPLOYMENT) IMAGE_FULL_REFERENCE=$(DATA_OPS_REPO):$(TAG) $(MAKE) -C . set-registry-image
+	@kubectl get deployment $(DATA_OPS_DEPLOYMENT) -o json | grep -v last | grep -qo debug.activate || DEPLOYMENT=$(DATA_OPS_DEPLOYMENT) $(MAKE) -C . add-debug-flag-agent
+	DEPLOYMENT=$(DATA_OPS_DEPLOYMENT) $(MAKE) -C . disable-frozen-modules
+	DEPLOYMENT=$(DATA_OPS_DEPLOYMENT) REPLICAS=$(CURRENT_DATA_OPS_REPLICAS) $(MAKE) scale
+
+revert-data-ops: cluster repo-$(DATA_OPS_REPO) delete-$(DATA_OPS_DEPLOYMENT) ## Reverts the data ops deployment to use the official image
+	DEPLOYMENT=$(DATA_OPS_DEPLOYMENT) IMAGE_FULL_REFERENCE=$(CONTAINER_REGISTRY_BASE)/$(DATA_OPS_REPO):$(FARMVIBES_AI_IMAGE_TAG) $(MAKE) set-registry-image
+	DEPLOYMENT=$(DATA_OPS_DEPLOYMENT) REPLICAS=$(CURRENT_DATA_OPS_REPLICAS) make scale
+
+local-worker: cluster local-worker-repo delete-$(WORKER_DEPLOYMENT) ## Builds and deploys a local WORKER image (enabling debug)
+	DEPLOYMENT=$(WORKER_DEPLOYMENT) IMAGE_FULL_REFERENCE=$(WORKER_REPO):$(TAG) $(MAKE) -C . set-registry-image
+	DEPLOYMENT=$(WORKER_DEPLOYMENT) $(MAKE) -C . disable-frozen-modules
+	DEPLOYMENT=$(WORKER_DEPLOYMENT) REPLICAS=$(CURRENT_WORKER_REPLICAS) make scale
+
+revert-worker: cluster repo-$(WORKER_REPO) delete-$(WORKER_DEPLOYMENT) ## Reverts the WORKER deployment to use the official image
+	DEPLOYMENT=$(WORKER_DEPLOYMENT) IMAGE_FULL_REFERENCE=$(CONTAINER_REGISTRY_BASE)/$(WORKER_REPO):$(FARMVIBES_AI_IMAGE_TAG) make set-registry-image
+	DEPLOYMENT=$(WORKER_DEPLOYMENT) REPLICAS=$(CURRENT_WORKER_REPLICAS) make scale
+
+local-cache: cluster local-cache-repo delete-$(CACHE_DEPLOYMENT) ## Builds and deploys a local CACHE image (enabling debug)
+	DEPLOYMENT=$(CACHE_DEPLOYMENT) IMAGE_FULL_REFERENCE=$(CACHE_REPO):$(TAG) $(MAKE) -C . set-registry-image
+	@kubectl get deployment $(CACHE_DEPLOYMENT) -o json | grep -v last | grep -qo debug.activate || DEPLOYMENT=$(CACHE_DEPLOYMENT) $(MAKE) -C . add-debug-flag-agent
+	DEPLOYMENT=$(CACHE_DEPLOYMENT) $(MAKE) -C . disable-frozen-modules
+	DEPLOYMENT=$(CACHE_DEPLOYMENT) REPLICAS=$(CURRENT_CACHE_REPLICAS) make scale
+
+revert-cache: cluster repo-$(CACHE_REPO) delete-$(CACHE_DEPLOYMENT) ## Reverts the CACHE deployment to use the official image
+	DEPLOYMENT=$(CACHE_DEPLOYMENT) IMAGE_FULL_REFERENCE=$(CONTAINER_REGISTRY_BASE)/$(CACHE_REPO):$(FARMVIBES_AI_IMAGE_TAG) make set-registry-image
+	DEPLOYMENT=$(CACHE_DEPLOYMENT) REPLICAS=$(CURRENT_CACHE_REPLICAS) make scale
+
+local-rest-api-orchestrator: cluster services-base
+	$(eval export PATH=$(HOME)/.config/farmvibes-ai:$(PATH))
+	docker build -t $(REST_API_REPO):$(TAG) -t $(ORCHESTRATOR_REPO):$(TAG) -f $(ROOT)/resources/docker/Dockerfile-api_orchestrator .
+
+local-cache-repo: cluster services-base
+	$(eval export PATH=$(HOME)/.config/farmvibes-ai:$(PATH))
+	docker build -t $(CACHE_REPO):$(TAG) -f $(ROOT)/resources/docker/Dockerfile-cache .
+
+local-worker-repo: cluster worker-base
+	$(eval export PATH=$(HOME)/.config/farmvibes-ai:$(PATH))
+	docker build -t $(WORKER_REPO):$(TAG) -f $(ROOT)/resources/docker/Dockerfile-worker .
+
+debug-rest-api: cluster local-rest-api  ## Starts listening to debug the REST API
+	DEPLOYMENT=$(REST_API_DEPLOYMENT) REPLICAS=1 make scale
+	kubectl port-forward deployments/$(REST_API_DEPLOYMENT) $(REST_API_DEBUG_PORT):$(CONTAINER_DEBUG_PORT)
+
+debug-orchestrator: cluster local-orchestrator  ## Starts listening to debug the ORCHESTRATOR
+	DEPLOYMENT=$(ORCHESTRATOR_DEPLOYMENT) REPLICAS=1 make scale
+	kubectl port-forward deployments/$(ORCHESTRATOR_DEPLOYMENT) $(ORCHESTRATOR_DEBUG_PORT):$(CONTAINER_DEBUG_PORT)
+
+debug-worker: cluster local-worker  ## Starts listening to debug the WORKER
+	@kubectl get deployment $(WORKER_DEPLOYMENT) -o json | grep -v last | grep -qo debug.activate || DEPLOYMENT=$(WORKER_DEPLOYMENT) $(MAKE) -C . add-debug-flag-agent
+	DEPLOYMENT=$(WORKER_DEPLOYMENT) REPLICAS=1 make scale
+	kubectl port-forward pod/`kubectl get pods -l app=$(WORKER_DEPLOYMENT) --field-selector status.phase=Running | awk '/Running/{ print $$1 }'` \
+		$(WORKER_DEBUG_PORT):$(CONTAINER_DEBUG_PORT)
+
+debug-cache: cluster local-cache  ## Starts listening to debug the CACHE
+	DEPLOYMENT=$(CACHE_DEPLOYMENT) REPLICAS=1 make scale
+	kubectl port-forward pod/`kubectl get pods -l app=$(CACHE_DEPLOYMENT) --field-selector status.phase=Running | awk '/Running/{ print $$1 }'` \
+		$(CACHE_DEBUG_PORT):$(CONTAINER_DEBUG_PORT)
+
+debug-data-ops: cluster local-data-ops  ## Starts listening to debug the DATA_OPS
+	DEPLOYMENT=$(DATA_OPS_DEPLOYMENT) REPLICAS=1 make scale
+	kubectl port-forward deployments/$(DATA_OPS_DEPLOYMENT) $(DATA_OPS_DEBUG_PORT):$(CONTAINER_DEBUG_PORT)
+
+clean: cluster revert clean-worker clean-orchestrator clean-rest-api clean-cache
+
+clean-cache: cluster revert-cache revert-worker ## Cleans up the cache image from the local docker "registry"
+	docker images | grep -E "$(CACHE_REPO)\\s+tmp.*" | awk '{ print $$3 }' | xargs docker rmi
+
+clean-worker: cluster revert-cache revert-worker ## Cleans up the worker image from the local docker "registry"
+	docker images | grep -E "$(WORKER_REPO)\\s+tmp.*" | awk '{ print $$3 }' | xargs docker rmi
+
+clean-orchestrator: cluster revert-rest-api revert-orchestrator ## Cleans up the orchestrator image from the local docker "registry"
+	docker images | grep -E "$(ORCHESTRATOR_REPO)\\s+tmp.*" | awk '{ print $$3 }' | xargs docker rmi
+
+clean-data-ops: cluster revert-rest-api revert-data-ops ## Cleans up the data-ops image from the local docker "registry"
+	docker images | grep -E "$(DATA_OPS_REPO)\\s+tmp.*" | awk '{ print $$3 }' | xargs docker rmi
+
+clean-rest-api: cluster revert-rest-api revert-orchestrator ## Cleans up the orchestrator image from the local docker "registry"
+	docker images | grep -E "$(REST_API_REPO)\\s+tmp.*" | awk '{ print $$3 }' | xargs docker rmi
+
+cluster:
+	$(eval export PATH=$(HOME)/.config/farmvibes-ai:$(PATH))
+	which k3d || $(build_cluster)
+	docker ps | grep -q farmvibes-ai || farmvibes-ai local start || $(build_cluster)
diff --git a/op_resources/average_model/pixel_average_model.onnx b/op_resources/average_model/pixel_average_model.onnx
diff --git a/op_resources/cdl_metadata/CDL_codes_names_colors.xls b/op_resources/cdl_metadata/CDL_codes_names_colors.xls