Merge branch 'staging'

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.12.5
+FROM python:3.12.6
 RUN echo "deb https://deb.debian.org/debian/ stable main" > /etc/apt/sources.list
 RUN apt-get update -y \
     && apt-get install -y build-essential python3-openstackclient vim\

docker-compose.bibigrid.host.yml

@@ -43,7 +43,7 @@ services:
 
   simplevm_client_filebeat:
     container_name: simplevm_client_filebeat
-    image: docker.elastic.co/beats/filebeat:8.15.0
+    image: docker.elastic.co/beats/filebeat:8.15.1
     network_mode: host  # Use the host network namespace
 
     env_file:

docker-compose.bibigrid.yml

@@ -44,7 +44,7 @@ services:
 
   # filebeat
   simplevm_filebeat:
-    image: docker.elastic.co/beats/filebeat:8.15.0
+    image: docker.elastic.co/beats/filebeat:8.15.1
     env_file:
       - .env
     volumes:

docker-compose.host.yml

@@ -30,7 +30,7 @@ services:
 
   simplevm_client_filebeat:
     container_name: simplevm_client_filebeat
-    image: docker.elastic.co/beats/filebeat:8.15.0
+    image: docker.elastic.co/beats/filebeat:8.15.1
     env_file:
       - .env
     volumes:

docker-compose.yml

@@ -30,7 +30,7 @@ services:
   # filebeat
   simplevm_client_filebeat:
     container_name: simplevm_client_filebeat
-    image: docker.elastic.co/beats/filebeat:8.15.0
+    image: docker.elastic.co/beats/filebeat:8.15.1
     env_file:
       - .env
     volumes:

portal_client.thrift

@@ -6,6 +6,24 @@ typedef i32 int
 /** The Version of the Portal-Client*/
 const string VERSION= '1.0.0'
 
+
+struct User {
+  1: optional string username,
+  2: required string user_id,
+  3: optional list<string> public_keys,
+}
+
+struct UserData {
+  2: required map<string, User> data = {},
+}
+
+struct VirtualMachineServerMetadata {
+  1: required string ip,
+  2: required string hashed_auth_token,
+  3: optional UserData userdata,
+}
+
+
 struct Backend {
     1: i64 id,
     2: string owner,
@@ -210,6 +228,13 @@ struct PlaybookResult {
     3: required string stderr
 }
 
+exception MetadataServerNotAvailableException {
+    1: string message
+}
+exception MetadataServerNotAllowedException{
+    1: string message
+}
+
 exception ResourceNotFoundException {
     /** Name already used. */
     1: string message
@@ -452,6 +477,8 @@ service VirtualMachineService {
      10:optional string research_environment
      11:optional list<string> additional_security_group_ids,
      12:optional string slurm_version,
+     13:optional string metadata_token,
+     14:optional string metadata_endpoint
 
     )
 
@@ -485,6 +512,10 @@ service VirtualMachineService {
     7:list<map<string,string>> volume_ids_path_new,
     8:list<map<string,string>> volume_ids_path_attach,
         9:optional list<string> additional_security_group_ids,
+             10:optional string metadata_token,
+                  11:optional string metadata_endpoint
+
+
 )  throws (1:NameAlreadyUsedException e,2:ResourceNotAvailableException r,3: ImageNotFoundException i,4: FlavorNotFoundException f,5:DefaultException d)
 
     /** Check if there is an instance with name */
@@ -542,6 +573,9 @@ service VirtualMachineService {
     ) throws (1:BackendNotFoundException b,2:DefaultException d)
 
 
+    void set_metadata_server_data(1:string ip,3:VirtualMachineServerMetadata metadata) throws (1:MetadataServerNotAvailableException m,2:MetadataServerNotAllowedException b)
+        void remove_metadata_server_data(1:string ip) throws (1:MetadataServerNotAvailableException m,2:MetadataServerNotAllowedException b)
+        void is_metadata_server_available() throws (1:MetadataServerNotAvailableException m,2:MetadataServerNotAllowedException b)
 
     /** Delete a backend*/
     void delete_backend(
@@ -825,4 +859,7 @@ service VirtualMachineService {
 
     throws (1:ServerNotFoundException e, 2: OpenStackConflictException c)
 
+
+
+
 }

requirements.txt

@@ -1,21 +1,21 @@
-setuptools==74.0.0
-thrift==0.20.0
-python-keystoneclient==5.4.0
-openstacksdk==3.1.0
+setuptools==75.1.0
+thrift==0.21.0
+openstacksdk==4.0.0
 deprecated==1.2.14
 Click==8.1.7
-ansible==9.9.0
+ansible==10.4.0
 flake8==7.1.1
-paramiko==3.4.1
+paramiko==3.5.0
 ruamel.yaml==0.18.6
 pyvim==3.0.3
 redis==5.0.8
 requests==2.32.3
 pyyaml==6.0.2
 pre-commit==3.8.0
-types-PyYAML==6.0.12.20240808
-sympy==1.13.2
+types-PyYAML==6.0.12.20240917
+sympy==1.13.3
 colorama==0.4.6
-types-redis==4.6.0.20240819
-gevent
-pytest
+gevent==24.2.1
+pytest==8.3.3
+types-redis==4.6.0.20240903
+oslo.utils==7.3.0

scripts/get_keys_from_metadata_server.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# Load the auth token from .metadata_config.env
+source ~/.metadata_config.env
+
+# Define the URL and machine IP
+URL="http://192.168.2.122:8000/metadata/192.168.2.74"
+AUTH_HEADER="auth_token: ${METADATA_ACCESS_TOKEN}"
+
+# Fetch the JSON response from the URL
+response=$(curl -s -X GET "$URL" -H "$AUTH_HEADER")
+
+# Extract the public_keys array from the JSON response
+public_keys=$(echo "$response" | jq -r '.public_keys[]')
+
+# Check if public_keys is empty
+if [ -z "$public_keys" ]; then
+  echo "No public keys found. authorized_keys file not updated."
+  exit 0
+fi
+
+# Ensure the .ssh directory and authorized_keys file exist
+mkdir -p ~/.ssh
+touch ~/.ssh/authorized_keys
+
+# Function to check if a key already exists in the authorized_keys file
+key_exists() {
+  grep -Fqx "$1" ~/.ssh/authorized_keys
+}
+
+# Add keys to authorized_keys if they don't already exist
+added_keys=0
+
+while IFS= read -r key; do
+  if ! key_exists "$key"; then
+    echo "$key" >> ~/.ssh/authorized_keys
+    ((added_keys++))
+  fi
+done <<< "$public_keys"
+
+if [ $added_keys -gt 0 ]; then
+  echo "$added_keys new public key(s) have been added to the authorized_keys file."
+else
+  echo "All public keys were already present. No changes made to authorized_keys file."
+fi

simple_vm_client/VirtualMachineHandler.py

@@ -12,6 +12,7 @@
 from simple_vm_client.util import thrift_converter
 from simple_vm_client.util.logger import setup_custom_logger
 
+from .metadata_connector.metadata_connector import MetadataConnector
 from .ttypes import (
     VM,
     Backend,
@@ -23,6 +24,7 @@
     PlaybookResult,
     ResearchEnvironmentTemplate,
     Snapshot,
+    VirtualMachineServerMetadata,
     Volume,
 )
 from .VirtualMachineService import Iface
@@ -37,6 +39,7 @@ def __init__(self, config_file: str):
         self.openstack_connector = OpenStackConnector(config_file=config_file)
         self.bibigrid_connector = BibigridConnector(config_file=config_file)
         self.forc_connector = ForcConnector(config_file=config_file)
+        self.metadata_connetor = MetadataConnector(config_file=config_file)
 
     def keyboard_interrupt_handler_playbooks(self) -> None:
         for k, v in self.forc_connector._active_playbooks.items():
@@ -50,6 +53,15 @@ def keyboard_interrupt_handler_playbooks(self) -> None:
             self.openstack_connector.delete_server(openstack_id=k)
         raise SystemExit(0)
 
+    def is_metadata_server_available(self):
+        return self.metadata_connetor.is_metadata_server_available()
+
+    def set_metadata_server_data(self, ip: str, metadata: VirtualMachineServerMetadata):
+        return self.metadata_connetor.set_metadata(ip=ip, metadata=metadata)
+
+    def remove_metadata_server_data(self, ip: str):
+        return self.metadata_connetor.remove_metadata(ip=ip)
+
     def get_images(self) -> list[Image]:
         images: list[Image] = thrift_converter.os_to_thrift_images(
             openstack_images=self.openstack_connector.get_images()
@@ -370,6 +382,8 @@ def start_server(
         research_environment: str,
         additional_security_group_ids: list[str],
         slurm_version: str = None,
+        metadata_token: str = None,
+        metadata_endpoint: str = None,
     ) -> str:
         if research_environment:
             research_environment_metadata = (
@@ -391,6 +405,8 @@ def start_server(
             research_environment_metadata=research_environment_metadata,
             additional_security_group_ids=additional_security_group_ids,
             slurm_version=slurm_version,
+            metadata_token=metadata_token,
+            metadata_endpoint=metadata_endpoint,
         )
 
     def start_server_with_custom_key(
@@ -403,6 +419,8 @@ def start_server_with_custom_key(
         volume_ids_path_new: list[dict[str, str]],
         volume_ids_path_attach: list[dict[str, str]],
         additional_security_group_ids: list[str],
+        metadata_token: str = None,
+        metadata_endpoint: str = None,
     ) -> str:
         if research_environment:
             research_environment_metadata = (
@@ -421,6 +439,8 @@ def start_server_with_custom_key(
             volume_ids_path_new=volume_ids_path_new,
             volume_ids_path_attach=volume_ids_path_attach,
             additional_security_group_ids=additional_security_group_ids,
+            metadata_token=metadata_token,
+            metadata_endpoint=metadata_endpoint,
         )
         self.forc_connector.set_vm_wait_for_playbook(
             openstack_id=openstack_id, private_key=private_key, name=servername