From 1fc7344d42707698ae2abe163daf2cc2db9bf33b Mon Sep 17 00:00:00 2001 From: catttam Date: Fri, 12 May 2023 10:25:35 +0200 Subject: [PATCH 1/4] Updated README to last version --- README.md | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d3d7c55..a5a86e9 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ ## Python OSCAR client + + [![Build](https://github.com/grycap/oscar_python/actions/workflows/main.yaml/badge.svg)](https://github.com/grycap/oscar_python/actions/workflows/main.yaml) ![PyPI](https://img.shields.io/pypi/v/oscar_python) @@ -8,6 +10,9 @@ This package provides a client to interact with OSCAR (https://oscar.grycap.net) ### Contents - [Python OSCAR client](#python-oscar-client) - [Contents](#contents) + - [Client](#client) + - [Initialize a client with basic authentication](#initialize-a-client-with-basic-authentication) + - [Initialize a client OIDC authentication](#initialize-a-client-oidc-authentication) - [Sample usage](#sample-usage) - [Client methods](#client-methods) - [Cluster methods](#cluster-methods) @@ -15,6 +20,43 @@ This package provides a client to interact with OSCAR (https://oscar.grycap.net) - [Logs methods](#logs-methods) - [Storage usage](#storage-usage) +### Client + +#### Initialize a client with basic authentication +``` python + options_basic_auth = {'cluster_id':'cluster-id', + 'endpoint':'https://cluster-endpoint', + 'user':'username', + 'password':'password', + 'ssl':'True'} + + client = Client(options = options_basic_auth) +``` +#### Initialize a client OIDC authentication + +If you want to use OIDC tokens to authenticate with EGI Check-In, you can use the [OIDC Agent](https://indigo-dc.gitbook.io/oidc-agent/) to create an account configuration for the EGI issuer (https://aai.egi.eu/auth/realms/egi/) and then initialize the client specifying the `shortname` of your account like follows. + +``` python + options_oidc_auth = {'cluster_id':'cluster-id', + 'endpoint':'https://cluster-endpoint', + 'shortname':'oidc-agent-shortname', + 'ssl':'True'} + + client = Client(options = options_oidc_auth) +``` + +If you already have a valid token, you can use the parameter `oidc_token` instead. + +``` python + options_oidc_auth = {'cluster_id':'cluster-id', + 'endpoint':'https://cluster-endpoint', + 'oidc_token':'token', + 'ssl':'True'} + + client = Client(options = options_oidc_auth) +``` +An example of using a generated token is if you want to use EGI Notebooks. Since you can't use oidc-agent on the Notebook, you can make use of the generated token that EGI provides on path `/var/run/secrets/egi.eu/access_token`. + ### Sample usage - Sample code that creates a client and gets information about the cluster @@ -22,7 +64,13 @@ This package provides a client to interact with OSCAR (https://oscar.grycap.net) ``` python from oscar_python.client import Client -client = Client("cluster-id","https://cluster-endpoint", "username", "password", True) +options_basic_auth = {'cluster_id':'cluster-id', + 'endpoint':'https://cluster-endpoint', + 'user':'username', + 'password':'password', + 'ssl':'True'} + +client = Client(options = options) # get the cluster information try: @@ -37,7 +85,13 @@ except Exception as err: ``` python from oscar_python.client import Client -client = Client("cluster-id","https://cluster-endpoint", "username", "password", True) +options_basic_auth = {'cluster_id':'cluster-id', + 'endpoint':'https://cluster-endpoint', + 'user':'username', + 'password':'password', + 'ssl':'True'} + +client = Client(options = options) try: client.create_service("/absolute_path/cowsay.yaml") From 9b97299e8f75c3672feb0216a52ec5bdc9f1774c Mon Sep 17 00:00:00 2001 From: catttam Date: Fri, 12 May 2023 10:26:52 +0200 Subject: [PATCH 2/4] Deleted comments --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index a5a86e9..349bc55 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,5 @@ ## Python OSCAR client - - [![Build](https://github.com/grycap/oscar_python/actions/workflows/main.yaml/badge.svg)](https://github.com/grycap/oscar_python/actions/workflows/main.yaml) ![PyPI](https://img.shields.io/pypi/v/oscar_python) From 0457fef1a843d0ef0862e6cee30505fe5664d330 Mon Sep 17 00:00:00 2001 From: catttam Date: Fri, 1 Sep 2023 10:25:54 +0200 Subject: [PATCH 3/4] Added create and update services with JSON definition --- oscar_python/client.py | 84 +++++++++++++++++++++++++++--------------- 1 file changed, 54 insertions(+), 30 deletions(-) diff --git a/oscar_python/client.py b/oscar_python/client.py index 8af18dd..00c1139 100644 --- a/oscar_python/client.py +++ b/oscar_python/client.py @@ -14,6 +14,7 @@ # limitations under the License. import json +import os import yaml import liboidcagent as agent import oscar_python._utils as utils @@ -99,44 +100,67 @@ def list_services(self): def get_service(self, name): return utils.make_request(self, _SVC_PATH+"/"+name, _GET) - def _apply_service(self, fdl_path, method): + def _check_fdl_definition(self, fdl_path): with open(fdl_path, "r") as read_fdl: fdl = self._parse_FDL_yaml(read_fdl) # Read FDL file and check correct format if fdl != ValueError: - for element in fdl["functions"]["oscar"]: - try: - svc = element[self.id] - except KeyError as err: - raise("FDL clusterID does not match current clusterID: {0}".format(err)) - # Check if service already exists when the function is called from create_service - if method == _POST: - svc_exists = utils.make_request(self, _SVC_PATH+"/"+svc["name"], _GET, handle=False) - if svc_exists.status_code == 200: - raise ValueError("A service with name '{0}' is already present on the cluster".format(svc["name"])) - try: - with open(svc["script"]) as s: - svc["script"] = s.read() - except IOError as err: - raise("Couldn't read script") - - # cpu parameter has to be string on the request - if type(svc["cpu"]) is int or type(svc["cpu"]) is float: svc["cpu"]= str(svc["cpu"]) - utils.make_request(self, _SVC_PATH, method, data=json.dumps(svc)) + try: + for element in fdl["functions"]["oscar"]: + try: + svc = element[self.id] + except KeyError as err: + raise("FDL clusterID does not match current clusterID: {0}".format(err)) + try: + with open(svc["script"]) as s: + svc["script"] = s.read() + except IOError as err: + raise("Couldn't read script") + + # cpu parameter has to be string on the request + if type(svc["cpu"]) is int or type(svc["cpu"]) is float: svc["cpu"]= str(svc["cpu"]) + + except ValueError as err: + print(err) + raise else: raise ValueError("Bad yaml format: {0}".format(fdl)) - - """ Create a service on the current cluster from a FDL file """ - def create_service(self, fdl_path): - return self._apply_service(fdl_path, _POST) - - """ Update a specific service """ - def update_service(self, name, fdl_path): + return svc + + """ Make the request to create a new service """ + def _apply_service(self, svc, method): + # Check if service already exists when the function is called from create_service + if method == _POST: + svc_exists = utils.make_request(self, _SVC_PATH+"/"+svc["name"], _GET, handle=False) + if svc_exists.status_code == 200: + raise ValueError("A service with name '{0}' is already present on the cluster".format(svc["name"])) + utils.make_request(self, _SVC_PATH, method, data=json.dumps(svc)) + + """ Create a service on the current cluster from a FDL file or a JSON definition """ + def create_service(self, service_definition): + if type(service_definition) is dict: + return self._apply_service(service_definition, _POST) + if os.path.isfile(service_definition): + try: + service = self._check_fdl_definition(service_definition) + except Exception: + raise + return self._apply_service(service, _POST) + + """ Update a specific service from a FDL file or a JSON definition """ + def update_service(self, name, new_service): # Check if service exists before update - svc = utils.make_request(self, _SVC_PATH+"/"+svc["name"], _GET, handle=False) + svc = utils.make_request(self, _SVC_PATH+"/"+name, _GET, handle=False) if svc.status_code != 200: raise ValueError("The service {0} is not present on the cluster".format(name)) - return self._apply_service(fdl_path, _PUT) + if type(new_service) is dict: + return self._apply_service(new_service, _PUT) + if os.path.isfile(new_service): + try: + service = self._check_fdl_definition(new_service) + except Exception: + raise + return self._apply_service(service, _PUT) """ Remove a specific service """ def remove_service(self, name): @@ -153,7 +177,7 @@ def run_service(self, name, **kwargs): send_data = utils.encode_input(exec_input) if "timeout" in kwargs.keys() and kwargs["timeout"]: - response = utils._make_request(self, _RUN_PATH+"/"+name, _POST, data=send_data, token=token, timeout=kwargs["timeout"]) + response = utils.make_request(self, _RUN_PATH+"/"+name, _POST, data=send_data, token=token, timeout=kwargs["timeout"]) else: response = utils.make_request(self, _RUN_PATH+"/"+name, _POST, data=send_data, token=token) From 98b6c321b32180cecd64f5034195420d1a1788ac Mon Sep 17 00:00:00 2001 From: catttam Date: Fri, 1 Sep 2023 10:27:58 +0200 Subject: [PATCH 4/4] Updated readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 349bc55..20a2910 100644 --- a/README.md +++ b/README.md @@ -135,13 +135,13 @@ services = client.list_services() # returns an http response or an HTTPError **create_service** ``` python # create a service -err = client.create_service("path_to_fdl") # returns nothing if the service is created or an error if something goes wrong +err = client.create_service("path_to_fdl" | "JSON_definition") # returns nothing if the service is created or an error if something goes wrong ``` **update_service** ``` python # update a service -err = client.update_service("service_name","path_to_fdl") # returns nothing if the service is created or an error if something goes wrong +err = client.update_service("service_name","path_to_fdl" | "JSON_definition") # returns nothing if the service is created or an error if something goes wrong ``` **remove_service**