From eb86fc054fbe1611f0c82bb454cbb393efc7fbdb Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Thu, 3 Nov 2022 06:24:56 -0400 Subject: [PATCH] support machine and context alias (#269) --- doc/batch.md | 8 ++++---- doc/context.md | 10 ++++----- dpdispatcher/base_context.py | 27 +++++++++++++++++++------ dpdispatcher/dp_cloud_server.py | 7 ++++--- dpdispatcher/dp_cloud_server_context.py | 7 ++++--- dpdispatcher/machine.py | 19 +++++++++++++---- tests/context.py | 1 + tests/test_class_machine_dispatch.py | 20 +++++++++++++++++- 8 files changed, 73 insertions(+), 26 deletions(-) diff --git a/doc/batch.md b/doc/batch.md index 634b9f4c..1f2f955b 100644 --- a/doc/batch.md +++ b/doc/batch.md @@ -49,12 +49,12 @@ One needs to make sure TORQUE has been setup in the remote server and the relate [IBM Spectrum LSF Suites](https://www.ibm.com/products/hpc-workload-management) is a comprehensive workload management solution used by HPCs. One needs to make sure LSF has been setup in the remote server and the related environment is activated. -## Lebesgue +## Bohrium -{dargs:argument}`batch_type `: `Lebesgue` +{dargs:argument}`batch_type `: `Bohrium` -Lebesgue is the cloud platform for scientific computing. -Read Lebesgue documentation for details. +Bohrium is the cloud platform for scientific computing. +Read Bohrium documentation for details. ## DistributedShell diff --git a/doc/context.md b/doc/context.md index a19cea26..8aeea3ae 100644 --- a/doc/context.md +++ b/doc/context.md @@ -28,13 +28,13 @@ It's suggested to generate [SSH keys](https://help.ubuntu.com/community/SSH/Open Note that `SSH` context is [non-login](https://www.gnu.org/software/bash/manual/html_node/Bash-Startup-Files.html), so `bash_profile` files will not be executed. -## Lebesgue +## Bohrium -{dargs:argument}`context_type `: `Lebesgue` +{dargs:argument}`context_type `: `Bohrium` -Lebesgue is the cloud platform for scientific computing. -Read Lebesgue documentation for details. -To use Lebesgue, one needs to provide necessary parameters in {dargs:argument}`remote_profile `. +Bohrium is the cloud platform for scientific computing. +Read Bohrium documentation for details. +To use Bohrium, one needs to provide necessary parameters in {dargs:argument}`remote_profile `. ## HDFS diff --git a/dpdispatcher/base_context.py b/dpdispatcher/base_context.py index cf46507e..dd780f2d 100644 --- a/dpdispatcher/base_context.py +++ b/dpdispatcher/base_context.py @@ -1,12 +1,16 @@ from abc import ABCMeta, abstractmethod from dargs import Argument -from typing import List +from typing import List, Optional, Tuple from dpdispatcher import dlog class BaseContext(metaclass=ABCMeta): subclasses_dict = {} options = set() + # alias: for subclasses_dict key + # notes: this attribute can be inherited + alias: Optional[Tuple[str, ...]] = tuple() + def __new__(cls, *args, **kwargs): if cls is BaseContext: subcls = cls.subclasses_dict[kwargs['context_type']] @@ -17,10 +21,12 @@ def __new__(cls, *args, **kwargs): def __init_subclass__(cls, **kwargs): super().__init_subclass__(**kwargs) - cls.subclasses_dict[cls.__name__]=cls - cls.subclasses_dict[cls.__name__.lower()]=cls - cls.subclasses_dict[cls.__name__.replace("Context", "")]=cls - cls.subclasses_dict[cls.__name__.lower().replace("context", "")]=cls + alias = [cls.__name__, *cls.alias] + for aa in alias: + cls.subclasses_dict[aa]=cls + cls.subclasses_dict[aa.lower()]=cls + cls.subclasses_dict[aa.replace("Context", "")]=cls + cls.subclasses_dict[aa.lower().replace("context", "")]=cls cls.options.add(cls.__name__) @classmethod @@ -77,12 +83,21 @@ def machine_arginfo(cls) -> Argument: Argument machine arginfo """ + alias = [] + for aa in cls.alias: + alias.extend(( + aa, + aa.lower(), + aa.replace("Context", ""), + aa.lower().replace("context", ""), + )) return Argument( cls.__name__, dict, sub_fields=cls.machine_subfields(), alias=[ cls.__name__.lower(), cls.__name__.replace("Context", ""), - cls.__name__.lower().replace("context", "") + cls.__name__.lower().replace("context", ""), + *alias, ]) @classmethod diff --git a/dpdispatcher/dp_cloud_server.py b/dpdispatcher/dp_cloud_server.py index d4dbb289..360ef8e8 100644 --- a/dpdispatcher/dp_cloud_server.py +++ b/dpdispatcher/dp_cloud_server.py @@ -16,7 +16,8 @@ """ -class DpCloudServer(Machine): +class Bohrium(Machine): + alias = ("Lebesgue", "DpCloudServer") def __init__(self, context): self.context = context self.input_data = context.remote_profile['input_data'].copy() @@ -201,5 +202,5 @@ def map_dp_job_state(status): # return self.context.check_file_exists(job_tag_finished) -class Lebesgue(DpCloudServer): - pass +DpCloudServer = Bohrium +Lebesgue = Bohrium diff --git a/dpdispatcher/dp_cloud_server_context.py b/dpdispatcher/dp_cloud_server_context.py index e6f450a7..dbc5048b 100644 --- a/dpdispatcher/dp_cloud_server_context.py +++ b/dpdispatcher/dp_cloud_server_context.py @@ -28,7 +28,8 @@ BUCKET_NAME = 'dpcloudserver' -class DpCloudServerContext(BaseContext): +class BohriumContext(BaseContext): + alias = ("DpCloudServerContext", 'LebesgueContext') def __init__(self, local_root, remote_root=None, @@ -269,7 +270,7 @@ def machine_subfields(cls) -> List[Argument]: ], doc=doc_remote_profile)] -class LebesgueContext(DpCloudServerContext): - pass +DpCloudServerContext = BohriumContext +LebesgueContext = BohriumContext # %% diff --git a/dpdispatcher/machine.py b/dpdispatcher/machine.py index 1ceff6e0..b3248451 100644 --- a/dpdispatcher/machine.py +++ b/dpdispatcher/machine.py @@ -3,7 +3,7 @@ from dpdispatcher.ssh_context import SSHSession import json from dargs import Argument, Variant -from typing import List, Union +from typing import List, Optional, Tuple import pathlib from dpdispatcher import dlog from dpdispatcher.base_context import BaseContext @@ -57,6 +57,9 @@ class Machine(metaclass=ABCMeta): subclasses_dict = {} options = set() + # alias: for subclasses_dict key + # notes: this attribute can be inherited + alias: Optional[Tuple[str, ...]] = tuple() def __new__(cls, *args, **kwargs): if cls is Machine: @@ -101,8 +104,10 @@ def bind_context(self, context): def __init_subclass__(cls, **kwargs): super().__init_subclass__(**kwargs) - cls.subclasses_dict[cls.__name__]=cls - cls.subclasses_dict[cls.__name__.lower()]=cls + alias = [cls.__name__, *cls.alias] + for aa in alias: + cls.subclasses_dict[aa]=cls + cls.subclasses_dict[aa.lower()]=cls cls.options.add(cls.__name__) # cls.subclasses.append(cls) @@ -357,9 +362,15 @@ def resources_arginfo(cls) -> Argument: Argument resources arginfo """ + alias = [] + for aa in cls.alias: + alias.extend(( + aa, + aa.lower(), + )) return Argument( cls.__name__, dict, sub_fields=cls.resources_subfields(), - alias=[cls.__name__.lower()] + alias=[cls.__name__.lower(), *alias] ) @classmethod diff --git a/tests/context.py b/tests/context.py index ff6e6360..28326fb5 100644 --- a/tests/context.py +++ b/tests/context.py @@ -19,6 +19,7 @@ from dpdispatcher.distributed_shell import DistributedShell from dpdispatcher.hdfs_context import HDFSContext from dpdispatcher.hdfs_cli import HDFS +from dpdispatcher.dp_cloud_server import Lebesgue from dpdispatcher.local_context import _identical_files try: diff --git a/tests/test_class_machine_dispatch.py b/tests/test_class_machine_dispatch.py index 846e91b6..3fe0a88b 100644 --- a/tests/test_class_machine_dispatch.py +++ b/tests/test_class_machine_dispatch.py @@ -14,7 +14,7 @@ from .context import LSF, Slurm, PBS, Shell from .context import Machine from .context import dargs -from .context import DistributedShell, HDFSContext +from .context import DistributedShell, HDFSContext, Lebesgue from .sample_class import SampleClass from dargs.dargs import ArgumentValueError @@ -195,6 +195,24 @@ def test_distributed_shell(self): ) self.assertIsInstance(machine, DistributedShell) + def test_lebesgue(self): + machine_dict = { + 'batch_type': 'Lebesgue', + 'context_type': 'LebesgueContext', + 'local_root':'./', + 'remote_root': './', + 'remote_profile': { + 'email': '114@514.com', + 'password': '114514', + 'program_id': 114514, + 'input_data': {}, + } + } + machine = Machine.load_from_dict( + machine_dict=machine_dict + ) + self.assertIsInstance(machine, Lebesgue) + class TestContextDispatch(unittest.TestCase): def setUp(self): self.maxDiff = None