From f7097b6551252e325b77889298dbd8c7db051728 Mon Sep 17 00:00:00 2001 From: Christian Muise Date: Tue, 10 May 2022 14:31:35 -0400 Subject: [PATCH 1/5] Getting started on the remote mirror. --- planutils/__init__.py | 4 ++++ planutils/package_installation.py | 9 ++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/planutils/__init__.py b/planutils/__init__.py index a4b9f25..315ab9a 100644 --- a/planutils/__init__.py +++ b/planutils/__init__.py @@ -116,6 +116,10 @@ def main(): from planutils.package_installation import run run(args.package, args.options) + elif 'remote' == args.command: + from planutils.package_installation import remote + remote(args.package, args.options) + elif 'list' == args.command: from planutils.package_installation import package_list package_list() diff --git a/planutils/package_installation.py b/planutils/package_installation.py index 97b2a28..a621993 100644 --- a/planutils/package_installation.py +++ b/planutils/package_installation.py @@ -4,7 +4,6 @@ from pathlib import Path from planutils import settings -from planutils import manifest_converter PACKAGES = {} @@ -193,3 +192,11 @@ def run(target, options): if not PACKAGES[target]["runnable"]: sys.exit(f"Package {target} is not executable") subprocess.run([Path(settings.PLANUTILS_PREFIX) / "packages" / target / "run"] + options) + + +def remote(target, options): + # TODO: check if the target is deplyed + # TODO: unpack the options and target to the right json + # TODO: run the remote command + # TODO: pack the results back to the client, including any new files + pass From ca8de9cbbc540932a67ddefe2df413e55b11e4c8 Mon Sep 17 00:00:00 2001 From: Christian Muise Date: Tue, 17 May 2022 23:45:52 -0400 Subject: [PATCH 2/5] Implemented the planutils remote redirect to paas --- planutils/__init__.py | 4 ++ planutils/package_installation.py | 87 ++++++++++++++++++++++++++++--- planutils/settings.py | 3 ++ 3 files changed, 88 insertions(+), 6 deletions(-) diff --git a/planutils/__init__.py b/planutils/__init__.py index 315ab9a..66e4711 100644 --- a/planutils/__init__.py +++ b/planutils/__init__.py @@ -85,6 +85,10 @@ def main(): parser_run.add_argument('package', help='package name') parser_run.add_argument('options', help='commandline options for the package', nargs="*") + parser_run = subparsers.add_parser('remote', help='run package remotely') + parser_run.add_argument('package', help='package name') + parser_run.add_argument('options', help='commandline options for the package', nargs="*") + parser_checkinstalled = subparsers.add_parser('check-installed', help='check if a package is installed') parser_checkinstalled.add_argument('package', help='package name') diff --git a/planutils/package_installation.py b/planutils/package_installation.py index a621993..674f643 100644 --- a/planutils/package_installation.py +++ b/planutils/package_installation.py @@ -1,5 +1,5 @@ -import json, os, glob, subprocess, sys +import json, os, glob, subprocess, sys, requests, time from collections import defaultdict from pathlib import Path @@ -195,8 +195,83 @@ def run(target, options): def remote(target, options): - # TODO: check if the target is deplyed - # TODO: unpack the options and target to the right json - # TODO: run the remote command - # TODO: pack the results back to the client, including any new files - pass + + # 1. check if the target is deployed + + package_url = settings.PAAS_SERVER + "/package" + r = requests.get(package_url) + remote_packages = r.json() + + remote_package = None + for p in remote_packages: + if p['package_name'] == target: + remote_package = p + break + + if (remote_package is None) or ('solve' not in remote_package['endpoint']['services']): + sys.exit(f"Package {target} is not remotely deployed") + + + # 2. unpack the options and target to the right json + + json_options = {} + remote_call = remote_package['endpoint']['services']['solve']['call'] + call_parts = remote_call.split(' ') + + args = {arg['name']: arg for arg in remote_package['endpoint']['services']['solve']['args']} + + if len(options) != len(call_parts)-1: + sys.exit(f"Call string does not match the remote call: {remote_package['endpoint']['services']['solve']['call']}") + + call_map = {} + for i, step in enumerate(call_parts[1:]): + if '{' == step[0] and '}' == step[-1]: + option = step[1:-1] + call_map[option] = options[i] + if option not in args: + sys.exit(f"Option {option} from call string is not defined in the remote call: {remote_call}") + if args[option]['type'] == 'file': + with open(options[i], 'r') as f: + json_options[option] = f.read() + else: + json_options[option] = options[i] + + rcall = remote_call + for k, v in call_map.items(): + rcall = rcall.replace('{' + k + '}', v) + print("\nMaking remote call: %s" % rcall) + + + # 3. run the remote command + + solve_url = '/'.join([settings.PAAS_SERVER, 'package', target, 'solve']) + r = requests.post(solve_url, json=json_options) + if r.status_code != 200: + sys.exit(f"Error running remote call: {r.text}") + + result_url = f"{settings.PAAS_SERVER}/{r.json()['result']}" + + # call every 0.5s until the result is ready + result = None + for _ in range(settings.PAAS_SERVER_LIMIT): + r = requests.get(result_url) + if (r.status_code == 200) and ('status' in r.json()) and (r.json()['status'] == 'ok'): + result = r.json()['result'] + break + time.sleep(0.5) + + if result is None: + sys.exit(f"Error running remote call: {r.text}") + + + # 4. unpack the results back to the client, including any new files + + result = r.json()['result'] + for fn in result['output']: + with open(fn, 'w') as f: + f.write(result['output'][fn]) + + print("\n\t{stdout}\n") + print(result['stdout']) + print("\n\t{stderr}\n") + print(result['stderr']) diff --git a/planutils/settings.py b/planutils/settings.py index 9f4ad37..f726861 100644 --- a/planutils/settings.py +++ b/planutils/settings.py @@ -8,6 +8,9 @@ SETTINGS_FILE = os.path.join(PLANUTILS_PREFIX, 'settings.json') +PAAS_SERVER = 'http://45.113.232.43:5001' +PAAS_SERVER_LIMIT = 100 + def load(): with open(SETTINGS_FILE, 'r') as f: settings = json.loads(f.read()) From e64e0afa64069646c2dcb39b7646f1cf19f24f45 Mon Sep 17 00:00:00 2001 From: nirlipo Date: Wed, 25 May 2022 13:30:24 +1000 Subject: [PATCH 3/5] check number of arguments instead of call parts --- planutils/package_installation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planutils/package_installation.py b/planutils/package_installation.py index 674f643..2488896 100644 --- a/planutils/package_installation.py +++ b/planutils/package_installation.py @@ -220,7 +220,7 @@ def remote(target, options): args = {arg['name']: arg for arg in remote_package['endpoint']['services']['solve']['args']} - if len(options) != len(call_parts)-1: + if len(options) != len(args)-1: sys.exit(f"Call string does not match the remote call: {remote_package['endpoint']['services']['solve']['call']}") call_map = {} From 30a8426de4123e164ea28bcd7a7720d81c5de92a Mon Sep 17 00:00:00 2001 From: nirlipo Date: Wed, 25 May 2022 16:24:49 +1000 Subject: [PATCH 4/5] fixed arguments check in remote call --- planutils/__init__.py | 6 +++--- planutils/package_installation.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/planutils/__init__.py b/planutils/__init__.py index 66e4711..6dbd33b 100644 --- a/planutils/__init__.py +++ b/planutils/__init__.py @@ -85,9 +85,9 @@ def main(): parser_run.add_argument('package', help='package name') parser_run.add_argument('options', help='commandline options for the package', nargs="*") - parser_run = subparsers.add_parser('remote', help='run package remotely') - parser_run.add_argument('package', help='package name') - parser_run.add_argument('options', help='commandline options for the package', nargs="*") + parser_remote = subparsers.add_parser('remote', help='run package remotely') + parser_remote.add_argument('package', help='package name') + parser_remote.add_argument('options', help='commandline options for the package', nargs="*") parser_checkinstalled = subparsers.add_parser('check-installed', help='check if a package is installed') parser_checkinstalled.add_argument('package', help='package name') diff --git a/planutils/package_installation.py b/planutils/package_installation.py index 2488896..4d01772 100644 --- a/planutils/package_installation.py +++ b/planutils/package_installation.py @@ -220,7 +220,7 @@ def remote(target, options): args = {arg['name']: arg for arg in remote_package['endpoint']['services']['solve']['args']} - if len(options) != len(args)-1: + if len(options) != len(args): sys.exit(f"Call string does not match the remote call: {remote_package['endpoint']['services']['solve']['call']}") call_map = {} From 70b49bdbed038dd4189d4d6f866167045fb85202 Mon Sep 17 00:00:00 2001 From: nirlipo Date: Wed, 25 May 2022 16:49:22 +1000 Subject: [PATCH 5/5] Added Remote-list command line option --- planutils/__init__.py | 7 +++++++ planutils/package_installation.py | 10 ++++++++++ 2 files changed, 17 insertions(+) diff --git a/planutils/__init__.py b/planutils/__init__.py index 6dbd33b..c7b6ed3 100644 --- a/planutils/__init__.py +++ b/planutils/__init__.py @@ -89,6 +89,9 @@ def main(): parser_remote.add_argument('package', help='package name') parser_remote.add_argument('options', help='commandline options for the package', nargs="*") + parser_remote_list = subparsers.add_parser('remote-list', help='list the available remote packages') + + parser_checkinstalled = subparsers.add_parser('check-installed', help='check if a package is installed') parser_checkinstalled.add_argument('package', help='package name') @@ -124,6 +127,10 @@ def main(): from planutils.package_installation import remote remote(args.package, args.options) + elif 'remote-list' == args.command: + from planutils.package_installation import package_remote_list + package_remote_list() + elif 'list' == args.command: from planutils.package_installation import package_list package_list() diff --git a/planutils/package_installation.py b/planutils/package_installation.py index 4d01772..f1d76b4 100644 --- a/planutils/package_installation.py +++ b/planutils/package_installation.py @@ -193,6 +193,16 @@ def run(target, options): sys.exit(f"Package {target} is not executable") subprocess.run([Path(settings.PLANUTILS_PREFIX) / "packages" / target / "run"] + options) +def package_remote_list(): + + package_url = settings.PAAS_SERVER + "/package" + r = requests.get(package_url) + remote_packages = r.json() + + print("\nDeployed:") + for p in remote_packages: + arguments = " ".join(f'{{{a["name"]}}} ' for a in p['endpoint']['services']['solve']['args']) + print(" %s: %s\n\tArguments: %s" % (p['package_name'], p['name'], arguments )) def remote(target, options):