merged main into current

.azure/.env.example

@@ -0,0 +1,5 @@
+AZURE_RESOUCE_GROUP=
+ACR_PASSWORD=
+AZURE_STORAGE_ACCOUNT_NAME=
+AZURE_STORAGE_ACCOUNT_KEY=
+AZURE_SSH_USERS=

.azure/README.md

@@ -0,0 +1,38 @@
+# Deploy docker iamge to ACI
+
+## Create ACI
+
+The below commands are to create new ACI by mounting fileshare storage.
+
+This creation can only be done one time if no need to change settings.
+
+- first replace variables in YAML
+
+```shell
+source ./.azure/.env
+
+ACR_PASSWORD=${ACR_PASSWORD} \
+AZURE_STORAGE_ACCOUNT_NAME=${AZURE_STORAGE_ACCOUNT_NAME} \
+AZURE_STORAGE_ACCOUNT_KEY=${AZURE_STORAGE_ACCOUNT_KEY} \
+SSH_USERS=$SSH_USERS \
+./.azure/aci-deploy.sh temp.yaml
+```
+
+- deploy
+
+```shell
+az login
+az container create --resource-group ${AZURE_RESOUCE_GROUP} --file temp.yaml
+
+# delete temp.yaml
+rm temp.yaml
+```
+
+## Restart
+
+when new image is deployed, ACI can be restarted.
+
+```shell
+# restart
+az container restart --resource-group ${AZURE_RESOUCE_GROUP} --name cbsurge-rapida
+```

.azure/aci-deploy.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+YAML_FILE="./.azure/aci-deploy.yaml"
+
+if [ -z "$AZURE_STORAGE_ACCOUNT_NAME" ]; then
+    echo "Error: AZURE_STORAGE_ACCOUNT_NAME is not set. Please provide a valid value."
+    exit 1
+fi
+
+if [ -z "$AZURE_STORAGE_ACCOUNT_KEY" ]; then
+    echo "Error: AZURE_STORAGE_ACCOUNT_KEY is not set. Please provide a valid value."
+    exit 1
+fi
+
+if [ -z "$ACR_PASSWORD" ]; then
+    echo "Error: ACR_PASSWORD is not set. Please provide a valid value."
+    exit 1
+fi
+
+if [ -z "$SSH_USERS" ]; then
+    echo "Error: SSH_USERS is not set. Please provide a valid value."
+    exit 1
+fi
+
+if [ -z "$1" ]; then
+    echo "Error: Output file name is not specified. Please provide an output file name as the first argument."
+    exit 1
+fi
+
+OUTPUT_FILE="$1"
+
+sed -e "s/{ACR_PASSWORD}/$ACR_PASSWORD/" \
+    -e "s/{AZURE_STORAGE_ACCOUNT_NAME}/$AZURE_STORAGE_ACCOUNT_NAME/" \
+    -e "s/{AZURE_STORAGE_ACCOUNT_KEY}/$AZURE_STORAGE_ACCOUNT_KEY/" \
+    -e "s/{SSH_USERS}/$SSH_USERS/" \
+    "$YAML_FILE" > $OUTPUT_FILE
+
+echo "$YAML_FILE was exported"

.azure/aci-deploy.yaml

@@ -0,0 +1,43 @@
+apiVersion: '2019-12-01'
+location: eastus
+name: cbsurge-rapida
+properties:
+  containers:
+  - name: cbsurge-rapida
+    properties:
+      environmentVariables:
+      - name: SSH_USERS
+        value: "{SSH_USERS}"
+      - name: PRODUCTION
+        value: "true"
+      image: undpgeohub.azurecr.io/undp-data/geo-cb-surge:main
+      ports:
+      - port: 22
+      - port: 80
+      resources:
+        requests:
+          cpu: 4
+          memoryInGB: 16
+      volumeMounts:
+      - mountPath: /home
+        name: fileshare-volume
+  imageRegistryCredentials:
+  - server: undpgeohub.azurecr.io
+    username: undpgeohub
+    password: {ACR_PASSWORD}
+  osType: Linux
+  restartPolicy: OnFailure
+  ipAddress:
+    type: Public
+    ports:
+      - port: 22
+      - port: 80
+    dnsNameLabel: cbsurge-rapida
+  volumes:
+  - name: fileshare-volume
+    azureFile:
+      sharename: cbrapida-aci
+      storageAccountName: {AZURE_STORAGE_ACCOUNT_NAME}
+      storageAccountKey: {AZURE_STORAGE_ACCOUNT_KEY}
+tags: {}
+type: Microsoft.ContainerInstance/containerGroups

.devcontainer/Dockerfile

@@ -6,3 +6,6 @@ RUN apt-get update && \
     apt-get install -y python3-pip pipenv gcc cmake libgeos-dev && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
+
+# install azure-cli
+RUN curl -sL https://aka.ms/InstallAzureCLIDeb | bash

.devcontainer/devcontainer.json

@@ -14,7 +14,7 @@
 		"ghcr.io/devcontainers/features/git:1": {}
 	},
 	//"onCreateCommand": "apt-get update && apt-get install -y python3-pip pipenv",
-	"postCreateCommand": "pipenv --python 3 && pipenv run pip install -r requirements.txt",
+	"postCreateCommand": "pipenv --python 3 && pipenv run pip install -e .",
 
 	//"postCreateCommand": "id",
 	"customizations": {

.dockerignore

@@ -7,3 +7,5 @@ docker-compose.yaml
 Dockerfile
 Pipfile
 Pipfile.lock
+.env.example
+.env

.env.example

@@ -1,4 +1,5 @@
 AZURE_STORAGE_CONNECTION_STRING=
 AZURE_STORAGE_CONTAINER=
 AZ_ROOT_FILE_PATH=
-LOCAL_DOWNLOAD_PATH=
+LOCAL_DOWNLOAD_PATH=
+SSH_USERS=

.github/workflows/ci.yml

@@ -55,11 +55,16 @@ jobs:
           mkdir -p "$CACHE_DIR"
           docker save ${{ steps.image_tag.outputs.TAG }} > ${{ steps.image_tag.outputs.TAR_PATH }}
 
+      - name: Install dependencies in container
+        run: |
+          docker run --rm -v ${{ github.workspace }}:/app -w /app --entrypoint /bin/bash ${{ steps.image_tag.outputs.TAG }} -c "
+            export LANG=C.UTF-8 && pipenv install -e .
+          "
       - name: Run tests in container
         run: |
           # Change owner of workspace to ubuntu user
           sudo chown -R 1000:1000 ${{ github.workspace }}
-          docker run --rm -v ${{ github.workspace }}:/app -w /app  ${{ steps.image_tag.outputs.TAG }} make test
+          docker run --rm -v ${{ github.workspace }}:/app -w /app --entrypoint /bin/bash ${{ steps.image_tag.outputs.TAG }} -c "make test"
 
   deploy-acr:
     name: Build and deploy to Azure Container Registry

Dockerfile

@@ -1,19 +1,46 @@
 # Use the GDAL image as the base
 FROM ghcr.io/osgeo/gdal:ubuntu-full-3.10.0
 
+ARG GROUPNAME="cbsurge"
+
 # Install necessary tools and Python packages
-RUN apt-get update && \
-    apt-get install -y python3-pip pipenv gcc cmake libgeos-dev && \
+RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - && \
+    apt-get update && \
+    apt-get install -y python3-pip pipenv \
+        gcc cmake libgeos-dev \
+        openssh-server \
+        ca-certificates curl gnupg nodejs && \
     apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
+    rm -rf /var/lib/apt/lists/* && \
+     npm install -g configurable-http-proxy
 
-WORKDIR /app
+# install azure-cli
+RUN curl -sL https://aka.ms/InstallAzureCLIDeb | bash
 
-COPY requirements.txt .
-RUN pipenv --python 3 && pipenv install -r requirements.txt
+RUN mkdir /var/run/sshd && \
+    echo 'PermitRootLogin no' >> /etc/ssh/sshd_config && \
+    echo 'PasswordAuthentication yes' >> /etc/ssh/sshd_config
+
+WORKDIR /app
 
 COPY . .
 
-RUN chmod +x cbsurge.sh
+# Create a group and set permissions for /app
+RUN groupadd ${GROUPNAME} && \
+    usermod -aG ${GROUPNAME} root && \
+    mkdir -p /app && \
+    chown -R :${GROUPNAME} /app && \
+    chmod -R g+rwx /app
+
+RUN chmod +x /app/create_user.sh
+RUN chmod +x /app/entrypoint.sh
+
+# install package
+ENV PIPENV_VENV_IN_PROJECT=1
+RUN pipenv install --dev --python 3 && \
+    pipenv run pip install -e .
+ENV VIRTUAL_ENV=/app/.venv
+
+EXPOSE 22
 
-CMD [ "./cbsurge.sh", "--help"]
+ENTRYPOINT ["/app/entrypoint.sh"]

Makefile

@@ -13,15 +13,15 @@ shell:
 	@echo "------------------------------------------------------------------"
 	@echo "Shelling in dev mode"
 	@echo "------------------------------------------------------------------"
-	docker compose -f docker-compose.yaml run cbsurge /bin/bash
+	docker compose -f docker-compose.yaml run --entrypoint /bin/bash cbsurge
 
 
 test:
 	@echo
 	@echo "------------------------------------------------------------------"
 	@echo "Execute test cases"
 	@echo "------------------------------------------------------------------"
-	pipenv run python -m pytest cbsurge/stats
+	pipenv run python -m pytest tests
 
 build:
 	@echo
@@ -30,6 +30,13 @@ build:
 	@echo "------------------------------------------------------------------"
 	docker compose -f docker-compose.yaml build
 
+up:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Launch docker containers"
+	@echo "------------------------------------------------------------------"
+	docker compose -f docker-compose.yaml up
+
 down:
 	@echo
 	@echo "------------------------------------------------------------------"