From f36f124534adffb2c37f7311ac7bf0a28fa98ad6 Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Thu, 17 Oct 2019 10:44:37 -0300 Subject: [PATCH 01/41] [ADD] .ini config¶meters --- faraday_agent_dispatcher/static/config.ini | 6 ++++++ faraday_agent_dispatcher/static/default_config.ini | 6 +++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/faraday_agent_dispatcher/static/config.ini b/faraday_agent_dispatcher/static/config.ini index e65464c7..a2e6317d 100644 --- a/faraday_agent_dispatcher/static/config.ini +++ b/faraday_agent_dispatcher/static/config.ini @@ -14,3 +14,9 @@ agent = G3mFSHRt8x7u2EHoWbbOftU7QtOHlW2aK6DXLAIOt9OqfRl4s03WVHzmSgL0YIxy cmd = python ./samples/scratch.py agent_name = agent +[varenvs] +var = SADASDASDASDASDASS + +[params] +p1 = int +p2 = str \ No newline at end of file diff --git a/faraday_agent_dispatcher/static/default_config.ini b/faraday_agent_dispatcher/static/default_config.ini index 2d65d54e..2484cba8 100644 --- a/faraday_agent_dispatcher/static/default_config.ini +++ b/faraday_agent_dispatcher/static/default_config.ini @@ -10,4 +10,8 @@ workspace = example cmd = python ./samples/scratch.py agent_name = agent -[tokens] \ No newline at end of file +[tokens] + +[varenvs] + +[params] \ No newline at end of file From fcc7b4242221872e085f679f9d4c47eee3e0bdcd Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Fri, 18 Oct 2019 14:57:40 -0300 Subject: [PATCH 02/41] [ADD] Argmuents and varenvs config + Sections static class --- faraday_agent_dispatcher/config.py | 10 +- faraday_agent_dispatcher/dispatcher.py | 115 +++++++++---- faraday_agent_dispatcher/static/config.ini | 9 +- tests/data/basic_executor.py | 7 +- tests/integration/faraday/test_execution.py | 19 +-- tests/unittests/test_agent_dispatcher.py | 177 +++++++++++--------- 6 files changed, 213 insertions(+), 124 deletions(-) diff --git a/faraday_agent_dispatcher/config.py b/faraday_agent_dispatcher/config.py index 413923f4..a731d02a 100644 --- a/faraday_agent_dispatcher/config.py +++ b/faraday_agent_dispatcher/config.py @@ -54,6 +54,10 @@ def save_config(filepath=None): instance.write(configfile) -TOKENS_SECTION = "tokens" -SERVER_SECTION = "server" -EXECUTOR_SECTION = "executor" +class Sections: + TOKENS = "tokens" + SERVER = "server" + EXECUTOR = "executor" + VARENVS = "varenvs" + PARAMS = "params" + diff --git a/faraday_agent_dispatcher/dispatcher.py b/faraday_agent_dispatcher/dispatcher.py index 5e2e1be7..8b0a3b07 100644 --- a/faraday_agent_dispatcher/dispatcher.py +++ b/faraday_agent_dispatcher/dispatcher.py @@ -33,42 +33,43 @@ ) import faraday_agent_dispatcher.logger as logging -from faraday_agent_dispatcher.config import instance as config, \ - EXECUTOR_SECTION, SERVER_SECTION, TOKENS_SECTION, save_config +from faraday_agent_dispatcher.config import instance as config, Sections, save_config logger = logging.get_logger() +logging.setup_logging() class Dispatcher: __control_dict = { - SERVER_SECTION: { + Sections.SERVER: { "host": control_host, "api_port": control_int, "websocket_port": control_int, "workspace": control_str }, - TOKENS_SECTION: { + Sections.TOKENS: { "registration": control_registration_token, "agent": control_agent_token }, - EXECUTOR_SECTION: { + Sections.EXECUTOR: { "cmd": control_str, "agent_name": control_str - } + }, + } def __init__(self, session, config_path=None): reset_config(filepath=config_path) self.control_config() self.config_path = config_path - self.host = config.get(SERVER_SECTION, "host") - self.api_port = config.get(SERVER_SECTION, "api_port") - self.websocket_port = config.get(SERVER_SECTION, "websocket_port") - self.workspace = config.get(SERVER_SECTION, "workspace") - self.agent_token = config[TOKENS_SECTION].get("agent", None) - self.executor_cmd = config.get(EXECUTOR_SECTION, "cmd") - self.agent_name = config.get(EXECUTOR_SECTION, "agent_name") + self.host = config.get(Sections.SERVER, "host") + self.api_port = config.get(Sections.SERVER, "api_port") + self.websocket_port = config.get(Sections.SERVER, "websocket_port") + self.workspace = config.get(Sections.SERVER, "workspace") + self.agent_token = config[Sections.TOKENS].get("agent", None) + self.executor_cmd = config.get(Sections.EXECUTOR, "cmd") + self.agent_name = config.get(Sections.EXECUTOR, "agent_name") self.session = session self.websocket = None self.websocket_token = None @@ -87,7 +88,7 @@ async def reset_websocket_token(self): async def register(self): if self.agent_token is None: - registration_token = self.agent_token = config.get(TOKENS_SECTION, "registration") + registration_token = self.agent_token = config.get(Sections.TOKENS, "registration") assert registration_token is not None, "The registration token is mandatory" token_registration_url = api_url(self.host, self.api_port, @@ -99,7 +100,7 @@ async def register(self): assert token_response.status == 201 token = await token_response.json() self.agent_token = token["token"] - config.set(TOKENS_SECTION, "agent", self.agent_token) + config.set(Sections.TOKENS, "agent", self.agent_token) save_config(self.config_path) except ClientResponseError as e: if e.status == 404: @@ -135,33 +136,83 @@ async def run_await(self): asyncio.create_task(self.run_once(data)) async def run_once(self, data:str= None): - # TODO Control data - logger.info('Running executor with data: %s'.format(data)) + logger.info('Parsing data: {}'.format(data)) data_dict = json.loads(data) if "action" in data_dict: if data_dict["action"] == "RUN": - process = await self.create_process() - tasks = [StdOutLineProcessor(process, self.session).process_f(), - StdErrLineProcessor(process).process_f(), - ] - - await asyncio.gather(*tasks) - await process.communicate() - assert process.returncode is not None - if process.returncode == 0: - logger.info("Executor finished successfully") - else: - logger.warning( - f"Executor finished with exit code {process.returncode}") + params = config.options(Sections.PARAMS).copy() + passed_params = data_dict['args'] + [params.remove(param) for param in config.defaults()] + # mandatoy_params_not_passed = [ + # not any([ + # param in passed_param # The param is not there + # for param in params # For all parameters + # ]) + # for passed_param in passed_params # For all parameter passed + #] + #assert not any(mandatoy_params_not_passed) + + all_accepted = all( + [ + any([ + param in passed_param # Control any available param + for param in params # was passed + ]) + for passed_param in passed_params # For all passed params + ]) + if not all_accepted: + logger.error("Unexpected argument passed") + mandatory_full = all( + [ + config.get(Sections.PARAMS, param) != "True" # All params is not mandatory + or any([ + param in passed_param for passed_param in passed_params # Or was passed + ]) + for param in params + ] + ) + if not mandatory_full: + logger.error("Mandatory argument not passed") + + if mandatory_full and all_accepted: + logger.info('Running executor') + process = await self.create_process(data_dict["args"]) + tasks = [StdOutLineProcessor(process, self.session).process_f(), + StdErrLineProcessor(process).process_f(), + ] + + await asyncio.gather(*tasks) + await process.communicate() + assert process.returncode is not None + if process.returncode == 0: + logger.info("Executor finished successfully") + else: + logger.warning( + f"Executor finished with exit code {process.returncode}") else: logger.info("Action unrecognized") else: logger.info("Data not contains action to do") - async def create_process(self): + async def create_process(self, args): + if args is None: + cmd = self.executor_cmd + elif isinstance(args, str): + logger.warning("Args from data received is a string") + cmd = self.executor_cmd + " --" + args + elif isinstance(args, list): + cmd = " --".join([self.executor_cmd] + args) + else: + logger.error("Args from data received has a not supported type") + raise ValueError("Args from data received has a not supported type") + import os + env = os.environ.copy() + for varenv in config.options(Sections.VARENVS): + if varenv not in config.defaults(): + env[varenv.upper()] = config.get(Sections.VARENVS,varenv) process = await asyncio.create_subprocess_shell( - self.executor_cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, env=env ) return process diff --git a/faraday_agent_dispatcher/static/config.ini b/faraday_agent_dispatcher/static/config.ini index a2e6317d..511d636c 100644 --- a/faraday_agent_dispatcher/static/config.ini +++ b/faraday_agent_dispatcher/static/config.ini @@ -18,5 +18,10 @@ agent_name = agent var = SADASDASDASDASDASS [params] -p1 = int -p2 = str \ No newline at end of file +out = True +count = False +spare = False +spaced_before = False +spaced_middle = False +err = False +fails = False \ No newline at end of file diff --git a/tests/data/basic_executor.py b/tests/data/basic_executor.py index ab32d8f6..e630b85b 100644 --- a/tests/data/basic_executor.py +++ b/tests/data/basic_executor.py @@ -13,6 +13,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import os import sys import json import argparse @@ -46,8 +47,8 @@ parser.add_argument('--spaced_middle', action='store_true', help='if set prints by stdout') parser.add_argument('--count', action='store', default=1, help='if set prints by stdout') args = parser.parse_args() - - if args.out: + omit_everything = os.getenv("DO_NOTHING", None) + if args.out and omit_everything is None: host_data_ = host_data.copy() host_data_['vulnerabilities'] = [vuln_data] data = dict(hosts=[host_data_]) @@ -61,6 +62,8 @@ elif args.out == "bad_json": del data["hosts"][0]["ip"] print(f"{json.dumps(data)}") + else: + print(omit_everything, file=sys.stderr) if args.err: print("Print by stderr", file=sys.stderr) diff --git a/tests/integration/faraday/test_execution.py b/tests/integration/faraday/test_execution.py index 694fc05a..6ed929b0 100644 --- a/tests/integration/faraday/test_execution.py +++ b/tests/integration/faraday/test_execution.py @@ -1,5 +1,4 @@ -from faraday_agent_dispatcher.config import instance as config, \ - EXECUTOR_SECTION, SERVER_SECTION, TOKENS_SECTION, save_config +from faraday_agent_dispatcher.config import instance as config, save_config, Sections from faraday_agent_dispatcher.utils.url_utils import api_url @@ -11,9 +10,9 @@ import subprocess import time -HOST = config.get(SERVER_SECTION, "host") -API_PORT = config.get(SERVER_SECTION, "api_port") -WS_PORT = config.get(SERVER_SECTION, "websocket_port") +HOST = config.get(Sections.SERVER, "host") +API_PORT = config.get(Sections.SERVER, "api_port") +WS_PORT = config.get(Sections.SERVER, "websocket_port") WORKSPACE = fuzzy_string(6).lower() # TODO FIX WHEN FARADAY ACCEPTS CAPITAL FIRST LETTER AGENT_NAME = fuzzy_string(6) @@ -50,11 +49,11 @@ def test_execute_agent(): token = res.json()['token'] # Config set up - config.set(TOKENS_SECTION, "registration", token) - config.remove_option(TOKENS_SECTION, "agent") - config.set(SERVER_SECTION, "workspace", WORKSPACE) - config.set(EXECUTOR_SECTION, "agent_name", AGENT_NAME) - config.set(EXECUTOR_SECTION, "cmd", "python ./basic_executor.py --out json") + config.set(Sections.TOKENS, "registration", token) + config.remove_option(Sections.TOKENS, "agent") + config.set(Sections.SERVER, "workspace", WORKSPACE) + config.set(Sections.EXECUTOR, "agent_name", AGENT_NAME) + config.set(Sections.EXECUTOR, "cmd", "python ./basic_executor.py --out json") save_config(CONFIG_DIR) # Init dispatcher! diff --git a/tests/unittests/test_agent_dispatcher.py b/tests/unittests/test_agent_dispatcher.py index 804728e2..2501c797 100644 --- a/tests/unittests/test_agent_dispatcher.py +++ b/tests/unittests/test_agent_dispatcher.py @@ -30,9 +30,7 @@ reset_config, save_config, instance as configuration, - SERVER_SECTION, - TOKENS_SECTION, - EXECUTOR_SECTION + Sections ) from tests.utils.text_utils import fuzzy_string @@ -41,51 +39,51 @@ @pytest.mark.parametrize('config_changes_dict', - [{"remove": {SERVER_SECTION: ["host"]}, + [{"remove": {Sections.SERVER: ["host"]}, "replace": {}}, # None error as default value - {"remove": {SERVER_SECTION: ["api_port"]}, + {"remove": {Sections.SERVER: ["api_port"]}, "replace": {}}, # None error as default value {"remove": {}, - "replace": {SERVER_SECTION: {"api_port": "Not a port number"}}, + "replace": {Sections.SERVER: {"api_port": "Not a port number"}}, "expected_exception": ValueError}, {"remove": {}, - "replace": {SERVER_SECTION: {"api_port": "6000"}}}, # None error as parse int - {"remove": {SERVER_SECTION: ["websocket_port"]}, + "replace": {Sections.SERVER: {"api_port": "6000"}}}, # None error as parse int + {"remove": {Sections.SERVER: ["websocket_port"]}, "replace": {}}, {"remove": {}, - "replace": {SERVER_SECTION: {"websocket_port": "Not a port number"}}, + "replace": {Sections.SERVER: {"websocket_port": "Not a port number"}}, "expected_exception": ValueError}, {"remove": {}, - "replace": {SERVER_SECTION: {"websocket_port": "9001"}}}, # None error as parse int - {"remove": {SERVER_SECTION: ["workspace"]}, + "replace": {Sections.SERVER: {"websocket_port": "9001"}}}, # None error as parse int + {"remove": {Sections.SERVER: ["workspace"]}, "replace": {}, "expected_exception": ValueError}, - {"remove": {TOKENS_SECTION: ["registration"]}, + {"remove": {Sections.TOKENS: ["registration"]}, "replace": {}, "expected_exception": ValueError}, {"remove": {}, - "replace": {TOKENS_SECTION: {"registration": "invalid_token"}}, + "replace": {Sections.TOKENS: {"registration": "invalid_token"}}, "expected_exception": ValueError}, {"remove": {}, - "replace": {TOKENS_SECTION: {"registration": " 46aasdje446aasdje446aa"}}, + "replace": {Sections.TOKENS: {"registration": " 46aasdje446aasdje446aa"}}, "expected_exception": ValueError}, {"remove": {}, - "replace": {TOKENS_SECTION: {"registration": "QWE46aasdje446aasdje446aa"}}}, + "replace": {Sections.TOKENS: {"registration": "QWE46aasdje446aasdje446aa"}}}, {"remove": {}, - "replace": {TOKENS_SECTION: {"agent": "invalid_token"}}, + "replace": {Sections.TOKENS: {"agent": "invalid_token"}}, "expected_exception": ValueError}, {"remove": {}, - "replace": {TOKENS_SECTION: { + "replace": {Sections.TOKENS: { "agent": " 46aasdje446aasdje446aa46aasdje446aasdje446aa46aasdje446aasdje" }}, "expected_exception": ValueError}, {"remove": {}, - "replace": {TOKENS_SECTION: { + "replace": {Sections.TOKENS: { "agent": "QWE46aasdje446aasdje446aaQWE46aasdje446aasdje446aaQWE46aasdje446"}}}, - {"remove": {EXECUTOR_SECTION: ["cmd"]}, + {"remove": {Sections.EXECUTOR: ["cmd"]}, "replace": {}, "expected_exception": ValueError}, - {"remove": {EXECUTOR_SECTION: ["agent_name"]}, + {"remove": {Sections.EXECUTOR: ["agent_name"]}, "replace": {}, "expected_exception": ValueError}, {"remove": {}, @@ -108,10 +106,10 @@ def test_basic_built(tmp_custom_config, config_changes_dict): async def test_start_and_register(test_config: FaradayTestConfig, tmp_default_config): # Config - configuration.set(SERVER_SECTION, "api_port", str(test_config.client.port)) - configuration.set(SERVER_SECTION, "host", test_config.client.host) - configuration.set(SERVER_SECTION, "workspace", test_config.workspace) - configuration.set(TOKENS_SECTION, "registration", test_config.registration_token) + configuration.set(Sections.SERVER, "api_port", str(test_config.client.port)) + configuration.set(Sections.SERVER, "host", test_config.client.host) + configuration.set(Sections.SERVER, "workspace", test_config.workspace) + configuration.set(Sections.TOKENS, "registration", test_config.registration_token) tmp_default_config.save() # Init and register it @@ -128,10 +126,10 @@ async def test_start_and_register(test_config: FaradayTestConfig, tmp_default_co async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default_config): # Config - configuration.set(SERVER_SECTION, "api_port", str(test_config.client.port)) - configuration.set(SERVER_SECTION, "host", test_config.client.host) - configuration.set(SERVER_SECTION, "workspace", test_config.workspace) - configuration.set(TOKENS_SECTION, "registration", "NotOk" * 5) + configuration.set(Sections.SERVER, "api_port", str(test_config.client.port)) + configuration.set(Sections.SERVER, "host", test_config.client.host) + configuration.set(Sections.SERVER, "workspace", test_config.workspace) + configuration.set(Sections.TOKENS, "registration", "NotOk" * 5) tmp_default_config.save() # Init and register it @@ -143,13 +141,14 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default @pytest.mark.skip def test_websocket(test_config: FaradayTestConfig, tmp_config): + #TODO WATCH OUT WHEN SKIP'LL BE FIXED text = fuzzy_string(15) file = f"/tmp/{fuzzy_string(8)}.txt" - configuration.set(SERVER_SECTION, "api_port", str(test_config.client.port)) - configuration.set(SERVER_SECTION, "workspace", test_config.workspace) - configuration.set(SERVER_SECTION, "websocket_port", str(test_config.websocket_port)) - configuration.set(SERVER_SECTION, "host", test_config.client.host) - configuration.set(EXECUTOR_SECTION, "cmd", f"echo {text} > {file}") + configuration.set(Sections.SERVER, "api_port", str(test_config.client.port)) + configuration.set(Sections.SERVER, "workspace", test_config.workspace) + configuration.set(Sections.SERVER, "websocket_port", str(test_config.websocket_port)) + configuration.set(Sections.SERVER, "host", test_config.client.host) + configuration.set(Sections.EXECUTOR, "cmd", f"echo {text} > {file}") tmp_config.save() dispatcher = Dispatcher(test_config.client.session, tmp_config.config_file_path) @@ -162,119 +161,145 @@ def test_websocket(test_config: FaradayTestConfig, tmp_config): @pytest.mark.parametrize('executor_options', [ - { + { # 0 "data": {"agent_id": 1}, - "args": [], "logs": [ {"levelname": "INFO", "msg": "Data not contains action to do"}, ] }, - { + { # 1 "data": {"action": "CUT", "agent_id": 1}, - "args": [], "logs": [ {"levelname": "INFO", "msg": "Action unrecognized"}, ] }, - { - "data": {"action": "RUN", "agent_id": 1}, - "args": ["out json"], + { # 2 + "data": {"action": "RUN", "agent_id": 1, "args": ["out json"]}, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "INFO", "msg": "Data sent to bulk create"}, {"levelname": "INFO", "msg": "Executor finished successfully"} ] }, - { - "data": {"action": "RUN", "agent_id": 1}, - "args": ["out json", "count 5"], + { # 3 + "data": {"action": "RUN", "agent_id": 1, "args": ["out json", "count 5"]}, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "ERROR", "msg": "JSON Parsing error: Extra data"}, {"levelname": "INFO", "msg": "Executor finished successfully"} ] }, - { - "data": {"action": "RUN", "agent_id": 1}, - "args": ["out json", "count 5", "spare"], + { # 4 + "data": {"action": "RUN", "agent_id": 1, "args": ["out json", "count 5", "spare"]}, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "INFO", "msg": "Data sent to bulk create", "min_count": 5}, {"levelname": "INFO", "msg": "Executor finished successfully"} ] }, - { - "data": {"action": "RUN", "agent_id": 1}, - "args": ["out json", "spaced_before"], + { # 5 + "data": {"action": "RUN", "agent_id": 1, "args": ["out json", "spaced_before"]}, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "INFO", "msg": "Executor finished successfully"} ] }, - { - "data": {"action": "RUN", "agent_id": 1}, - "args": ["out json", "spaced_middle", "count 5", "spare"], + { # 6 + "data": {"action": "RUN", "agent_id": 1, "args": ["out json", "spaced_middle", + "count 5", "spare"]}, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "INFO", "msg": "Data sent to bulk create", "max_count": 1}, {"levelname": "INFO", "msg": "Executor finished successfully"} ] }, - { - "data": {"action": "RUN", "agent_id": 1}, - "args": ["out bad_json"], + { # 7 + "data": {"action": "RUN", "agent_id": 1, "args": ["out bad_json"]}, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "ERROR", - "msg": "Invalid data supplied by the executor to the bulk create endpoint. Server responded: "}, + "msg": "Invalid data supplied by the executor to the bulk create endpoint. " + "Server responded: "}, {"levelname": "INFO", "msg": "Executor finished successfully"} ] }, - { - "data": {"action": "RUN", "agent_id": 1}, - "args": ["out str"], + { # 8 + "data": {"action": "RUN", "agent_id": 1, "args": ["out str"]}, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "ERROR", "msg": "JSON Parsing error: Expecting value"}, {"levelname": "INFO", "msg": "Executor finished successfully"} ] }, - { - "data": {"action": "RUN", "agent_id": 1}, - "args": ["err"], + { # 9 + "data": {"action": "RUN", "agent_id": 1, "args": ["out none", "err"]}, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "DEBUG", "msg": "Print by stderr"}, {"levelname": "INFO", "msg": "Executor finished successfully"} ] }, - { - "data": {"action": "RUN", "agent_id": 1}, - "args": ["fails"], + { # 10 + "data": {"action": "RUN", "agent_id": 1, "args": ["out none", "fails"]}, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "WARNING", "msg": "Executor finished with exit code 1"}, ] }, - { - "data": {"action": "RUN", "agent_id": 1}, - "args": ["err", "fails"], + { # 11 + "data": {"action": "RUN", "agent_id": 1, "args": ["out none", "err", "fails"]}, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "DEBUG", "msg": "Print by stderr"}, {"levelname": "WARNING", "msg": "Executor finished with exit code 1"}, ] }, + { # 12 + "data": {"action": "RUN", "agent_id": 1, "args": ["out json"]}, + "logs": [ + {"levelname": "INFO", "msg": "Running executor"}, + {"levelname": "INFO", "msg": "Data sent to bulk create", "max_count": 0, + "min_count": 0}, + {"levelname": "INFO", "msg": "Executor finished successfully"} + ], + "varenvs": {"DO_NOTHING": "True"} + }, + { # 13 + "data": {"action": "RUN", "agent_id": 1, "args": ["err", "fails"]}, + "logs": [ + {"levelname": "INFO", "msg": "Running executor", "max_count": 0, + "min_count": 0}, + {"levelname": "ERROR", "msg": "Mandatory argument not passed"}, + ] + }, + { # 14 + "data": {"action": "RUN", "agent_id": 1, "args": ["out json", "WTF"]}, + "logs": [ + {"levelname": "INFO", "msg": "Running executor", "max_count": 0, + "min_count": 0}, + {"levelname": "INFO", "msg": "Data sent to bulk create", "max_count": 0, + "min_count": 0}, + {"levelname": "INFO", "msg": "Executor finished successfully", "max_count": 0, + "min_count": 0}, + {"levelname": "ERROR", "msg": "Unexpected argument passed"}, + ] + }, + ]) async def test_run_once(test_config: FaradayTestConfig, tmp_default_config, test_logger_handler, executor_options): # Config - configuration.set(SERVER_SECTION, "api_port", str(test_config.client.port)) - configuration.set(SERVER_SECTION, "host", test_config.client.host) - configuration.set(SERVER_SECTION, "workspace", test_config.workspace) - configuration.set(TOKENS_SECTION, "registration", test_config.registration_token) - configuration.set(TOKENS_SECTION, "agent", test_config.agent_token) - configuration.set(EXECUTOR_SECTION, "cmd", " --".join(["python ./basic_executor.py"] + - executor_options["args"])) + configuration.set(Sections.SERVER, "api_port", str(test_config.client.port)) + configuration.set(Sections.SERVER, "host", test_config.client.host) + configuration.set(Sections.SERVER, "workspace", test_config.workspace) + configuration.set(Sections.TOKENS, "registration", test_config.registration_token) + configuration.set(Sections.TOKENS, "agent", test_config.agent_token) + configuration.set(Sections.EXECUTOR, "cmd", "python ../data/basic_executor.py") + configuration.set(Sections.PARAMS, "out", "True") + [configuration.set(Sections.PARAMS, param, "False") for param in [ + "count", "spare", "spaced_before", "spaced_middle", "err", "fails"]] + if "varenvs" in executor_options: + for varenv in executor_options["varenvs"]: + configuration.set(Sections.VARENVS, varenv, executor_options["varenvs"][varenv]) tmp_default_config.save() # Init and register it @@ -287,3 +312,5 @@ async def test_run_once(test_config: FaradayTestConfig, tmp_default_config, test assert max_count >= \ len(list(filter(lambda x: x.levelname == l["levelname"] and l["msg"] in x.message, history))) >= \ min_count + + From c2356e1103f43acf9fdefbe923bfca25d60f79f6 Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Fri, 18 Oct 2019 15:31:06 -0300 Subject: [PATCH 03/41] [FIX] ENVVAR for CI tests --- .gitlab-ci.yml | 1 + tests/unittests/test_agent_dispatcher.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f438906a..6a52cf4a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,6 +9,7 @@ variables: FARADAY_PASSWORD: custom_pass FARADAY_EMAIL: test@test.com FARADAY_REF: white/dev + EXECUTOR_DIR: ./basic_executor.py cache: paths: diff --git a/tests/unittests/test_agent_dispatcher.py b/tests/unittests/test_agent_dispatcher.py index 2501c797..6f64a9c4 100644 --- a/tests/unittests/test_agent_dispatcher.py +++ b/tests/unittests/test_agent_dispatcher.py @@ -288,12 +288,15 @@ def test_websocket(test_config: FaradayTestConfig, tmp_config): ]) async def test_run_once(test_config: FaradayTestConfig, tmp_default_config, test_logger_handler, executor_options): # Config + + executor_path = "../data/basic_executor.py" if "EXECUTOR_DIR" not in os.environ else os.environ["EXECUTOR_DIR"] + configuration.set(Sections.SERVER, "api_port", str(test_config.client.port)) configuration.set(Sections.SERVER, "host", test_config.client.host) configuration.set(Sections.SERVER, "workspace", test_config.workspace) configuration.set(Sections.TOKENS, "registration", test_config.registration_token) configuration.set(Sections.TOKENS, "agent", test_config.agent_token) - configuration.set(Sections.EXECUTOR, "cmd", "python ../data/basic_executor.py") + configuration.set(Sections.EXECUTOR, "cmd", "python {}".format(executor_path)) configuration.set(Sections.PARAMS, "out", "True") [configuration.set(Sections.PARAMS, param, "False") for param in [ "count", "spare", "spaced_before", "spaced_middle", "err", "fails"]] From bb6311b202837889b55fa6a38baa501fa3a8f657 Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Fri, 1 Nov 2019 15:05:36 -0300 Subject: [PATCH 04/41] [FIX] Integration tests --- faraday_agent_dispatcher/dispatcher.py | 4 ++-- tests/integration/faraday/test_execution.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/faraday_agent_dispatcher/dispatcher.py b/faraday_agent_dispatcher/dispatcher.py index ef1119b6..6781d8cd 100644 --- a/faraday_agent_dispatcher/dispatcher.py +++ b/faraday_agent_dispatcher/dispatcher.py @@ -140,7 +140,7 @@ async def run_once(self, data:str= None): if "action" in data_dict: if data_dict["action"] == "RUN": params = config.options(Sections.PARAMS).copy() - passed_params = data_dict['args'] + passed_params = data_dict['args'] if 'args' in data_dict else {} [params.remove(param) for param in config.defaults()] # mandatoy_params_not_passed = [ # not any([ @@ -175,7 +175,7 @@ async def run_once(self, data:str= None): if mandatory_full and all_accepted: logger.info('Running executor') - process = await self.create_process(data_dict["args"]) + process = await self.create_process(passed_params) tasks = [StdOutLineProcessor(process, self.session).process_f(), StdErrLineProcessor(process).process_f(), ] diff --git a/tests/integration/faraday/test_execution.py b/tests/integration/faraday/test_execution.py index 6d609656..86913af9 100644 --- a/tests/integration/faraday/test_execution.py +++ b/tests/integration/faraday/test_execution.py @@ -97,7 +97,10 @@ def test_execute_agent(): assert host_dict["total_rows"] == 1 host = host_dict["rows"][0]["value"] for key in host_data: - assert host[key] == host_data[key] + if key == "hostnames": + assert set(host[key]) == set(host_data[key]) + else: + assert host[key] == host_data[key] assert host["vulns"] == 1 res = session.get(api_url(HOST, API_PORT, postfix=f'/_api/v2/ws/{WORKSPACE}/vulns')) From be2aeb8e1cd865a85c47b3361183a80085157c7e Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Fri, 1 Nov 2019 18:17:32 +0000 Subject: [PATCH 05/41] Args ci mod; logs folder fix; args by enviroment vars --- .gitlab-ci.yml | 12 +++-- faraday_agent_dispatcher/dispatcher.py | 24 +++------ faraday_agent_dispatcher/example_config.ini | 4 ++ .../utils/control_values_utils.py | 2 + tests/data/basic_executor.py | 36 +++++++------ tests/integration/faraday/test_execution.py | 3 +- tests/unittests/test_agent_dispatcher.py | 53 ++++++++++--------- tests/utils/testing_faraday_server.py | 9 +++- 8 files changed, 75 insertions(+), 68 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c68be9c5..73d9e208 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -38,15 +38,17 @@ unit_tests: - pip install pytest pytest-cov pytest-aiohttp - python setup.py install - mkdir run_from - - cd run_from && pytest ../tests/unittests --capture=sys -v --cov=../faraday_agent_dispatcher --color=yes --disable-warnings + - cd run_from && mkdir logs && pytest ../tests/unittests --capture=sys -v --cov=../faraday_agent_dispatcher --color=yes --disable-warnings artifacts: - when: on_failure - paths: - - dist/* + when: on_failure + paths: + - dist/* + - run_from/logs/* + expire_in: 7 days except: variables: - $INTEGRATION - - $CI_COMMIT_REF_NAME =~ /^.*ci.*$/ + - $CI_COMMIT_REF_NAME =~ /^.*ci-int.*$/ integration_faraday: image: python:3 diff --git a/faraday_agent_dispatcher/dispatcher.py b/faraday_agent_dispatcher/dispatcher.py index 6781d8cd..37963ee9 100644 --- a/faraday_agent_dispatcher/dispatcher.py +++ b/faraday_agent_dispatcher/dispatcher.py @@ -15,6 +15,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import os import json import asyncio @@ -142,14 +143,6 @@ async def run_once(self, data:str= None): params = config.options(Sections.PARAMS).copy() passed_params = data_dict['args'] if 'args' in data_dict else {} [params.remove(param) for param in config.defaults()] - # mandatoy_params_not_passed = [ - # not any([ - # param in passed_param # The param is not there - # for param in params # For all parameters - # ]) - # for passed_param in passed_params # For all parameter passed - #] - #assert not any(mandatoy_params_not_passed) all_accepted = all( [ @@ -195,23 +188,18 @@ async def run_once(self, data:str= None): logger.info("Data not contains action to do") async def create_process(self, args): - if args is None: - cmd = self.executor_cmd - elif isinstance(args, str): - logger.warning("Args from data received is a string") - cmd = self.executor_cmd + " --" + args - elif isinstance(args, list): - cmd = " --".join([self.executor_cmd] + args) + env = os.environ.copy() + if isinstance(args, dict): + for k in args: + env[k.upper()] = str(args[k]) else: logger.error("Args from data received has a not supported type") raise ValueError("Args from data received has a not supported type") - import os - env = os.environ.copy() for varenv in config.options(Sections.VARENVS): if varenv not in config.defaults(): env[varenv.upper()] = config.get(Sections.VARENVS,varenv) process = await asyncio.create_subprocess_shell( - cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, env=env + self.executor_cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, env=env ) return process diff --git a/faraday_agent_dispatcher/example_config.ini b/faraday_agent_dispatcher/example_config.ini index fe52d254..c9b126eb 100644 --- a/faraday_agent_dispatcher/example_config.ini +++ b/faraday_agent_dispatcher/example_config.ini @@ -13,3 +13,7 @@ agent_name = unnamed_agent ; To get your registration token, visit http://localhost:5985/#/admin/agents, copy ; the value and uncomment the line ; registration = + +[varenvs] + +[params] \ No newline at end of file diff --git a/faraday_agent_dispatcher/utils/control_values_utils.py b/faraday_agent_dispatcher/utils/control_values_utils.py index d916f6af..6464c0a0 100644 --- a/faraday_agent_dispatcher/utils/control_values_utils.py +++ b/faraday_agent_dispatcher/utils/control_values_utils.py @@ -1,5 +1,7 @@ def control_int(field_name,value): + if value is None: + raise ValueError(f"Trying to parse {field_name} with None value and should be an int") try: int(value) except ValueError: diff --git a/tests/data/basic_executor.py b/tests/data/basic_executor.py index e630b85b..e6056eb2 100644 --- a/tests/data/basic_executor.py +++ b/tests/data/basic_executor.py @@ -16,7 +16,6 @@ import os import sys import json -import argparse host_data = { @@ -37,35 +36,40 @@ 'refs': ['CVE-1234'] } +import argparse # TODO REMOVE WHEN FARADAY SENDS ARGS + if __name__ == '__main__': + ##### TODO REMOVE WHEN FARADAY SENDS ARGS parser = argparse.ArgumentParser() parser.add_argument('--out', action='store', help='if set prints by stdout') - parser.add_argument('--err', action='store_true', help='if set prints by stderr') - parser.add_argument('--fails', action='store_true', help='if true fails') - parser.add_argument('--spaced_before', action='store_true', help='if set prints by stdout') - parser.add_argument('--spare', action='store_true', help='if set prints by stdout') - parser.add_argument('--spaced_middle', action='store_true', help='if set prints by stdout') - parser.add_argument('--count', action='store', default=1, help='if set prints by stdout') args = parser.parse_args() + ##### TODO REMOVE WHEN FARADAY SENDS ARGS + out = os.getenv("OUT", args.out or None) + count = os.getenv("COUNT", 1) + err = os.getenv("ERR") is not None + fails = os.getenv("FAILS") is not None + spaced_before = os.getenv("SPACED_BEFORE") is not None + spaced_middle = os.getenv("SPACED_MIDDLE") is not None + spare = os.getenv("SPARE") is not None omit_everything = os.getenv("DO_NOTHING", None) - if args.out and omit_everything is None: + if out and omit_everything is None: host_data_ = host_data.copy() host_data_['vulnerabilities'] = [vuln_data] data = dict(hosts=[host_data_]) - if args.out == "json": - prefix = '\n' if args.spaced_before else '' - suffix = '\n' if args.spaced_middle else '' - suffix += ('\n' if args.spare else '').join([''] + [json.dumps(data) for _ in range(int(args.count) - 1)]) + if out == "json": + prefix = '\n' if spaced_before else '' + suffix = '\n' if spaced_middle else '' + suffix += ('\n' if spare else '').join([''] + [json.dumps(data) for _ in range(int(count) - 1)]) print(f"{prefix}{json.dumps(data)}{suffix}") - elif args.out == "str": + elif out == "str": print("NO JSON OUTPUT") - elif args.out == "bad_json": + elif out == "bad_json": del data["hosts"][0]["ip"] print(f"{json.dumps(data)}") else: print(omit_everything, file=sys.stderr) - if args.err: + if err: print("Print by stderr", file=sys.stderr) - if args.fails: + if fails: sys.exit(1) \ No newline at end of file diff --git a/tests/integration/faraday/test_execution.py b/tests/integration/faraday/test_execution.py index 86913af9..eb7257cc 100644 --- a/tests/integration/faraday/test_execution.py +++ b/tests/integration/faraday/test_execution.py @@ -56,7 +56,6 @@ def test_execute_agent(): config.remove_option(Sections.TOKENS, "agent") config.set(Sections.SERVER, "workspace", WORKSPACE) config.set(Sections.EXECUTOR, "agent_name", AGENT_NAME) - config.set(Sections.EXECUTOR, "cmd", "python ./basic_executor.py --out json") path_to_basic_executor = ( Path(__file__).parent.parent.parent / 'data' / 'basic_executor.py' @@ -94,7 +93,7 @@ def test_execute_agent(): # Test results res = session.get(api_url(HOST, API_PORT, postfix=f'/_api/v2/ws/{WORKSPACE}/hosts')) host_dict = res.json() - assert host_dict["total_rows"] == 1 + assert host_dict["total_rows"] == 1, (res.text, host_dict) host = host_dict["rows"][0]["value"] for key in host_data: if key == "hostnames": diff --git a/tests/unittests/test_agent_dispatcher.py b/tests/unittests/test_agent_dispatcher.py index b44bdb51..67b9b6b7 100644 --- a/tests/unittests/test_agent_dispatcher.py +++ b/tests/unittests/test_agent_dispatcher.py @@ -36,21 +36,24 @@ from tests.utils.text_utils import fuzzy_string from tests.utils.testing_faraday_server import FaradayTestConfig, test_config, tmp_custom_config, tmp_default_config, \ - test_logger_handler + test_logger_handler, test_logger_folder @pytest.mark.parametrize('config_changes_dict', [{"remove": {Sections.SERVER: ["host"]}, - "replace": {}}, # None error as default value + "replace": {}, + "expected_exception": ValueError}, {"remove": {Sections.SERVER: ["api_port"]}, - "replace": {}}, # None error as default value + "replace": {}, + "expected_exception": ValueError}, {"remove": {}, "replace": {Sections.SERVER: {"api_port": "Not a port number"}}, "expected_exception": ValueError}, {"remove": {}, "replace": {Sections.SERVER: {"api_port": "6000"}}}, # None error as parse int {"remove": {Sections.SERVER: ["websocket_port"]}, - "replace": {}}, + "replace": {}, + "expected_exception": ValueError}, {"remove": {}, "replace": {Sections.SERVER: {"websocket_port": "Not a port number"}}, "expected_exception": ValueError}, @@ -177,7 +180,7 @@ def test_websocket(test_config: FaradayTestConfig, tmp_config): ] }, { # 2 - "data": {"action": "RUN", "agent_id": 1, "args": ["out json"]}, + "data": {"action": "RUN", "agent_id": 1, "args": {"out": "json"}}, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "INFO", "msg": "Data sent to bulk create"}, @@ -185,7 +188,7 @@ def test_websocket(test_config: FaradayTestConfig, tmp_config): ] }, { # 3 - "data": {"action": "RUN", "agent_id": 1, "args": ["out json", "count 5"]}, + "data": {"action": "RUN", "agent_id": 1, "args": {"out": "json", "count": "5"}}, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "ERROR", "msg": "JSON Parsing error: Extra data"}, @@ -193,7 +196,8 @@ def test_websocket(test_config: FaradayTestConfig, tmp_config): ] }, { # 4 - "data": {"action": "RUN", "agent_id": 1, "args": ["out json", "count 5", "spare"]}, + "data": {"action": "RUN", "agent_id": 1, "args": {"out": "json", "count": "5", + "spare": "T"}}, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "INFO", "msg": "Data sent to bulk create", "min_count": 5}, @@ -201,15 +205,15 @@ def test_websocket(test_config: FaradayTestConfig, tmp_config): ] }, { # 5 - "data": {"action": "RUN", "agent_id": 1, "args": ["out json", "spaced_before"]}, + "data": {"action": "RUN", "agent_id": 1, "args": {"out": "json", "spaced_before":"T"}}, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "INFO", "msg": "Executor finished successfully"} ] }, { # 6 - "data": {"action": "RUN", "agent_id": 1, "args": ["out json", "spaced_middle", - "count 5", "spare"]}, + "data": {"action": "RUN", "agent_id": 1, "args": {"out": "json", "spaced_middle": "T", + "count": "5", "spare": "T"}}, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "INFO", "msg": "Data sent to bulk create", "max_count": 1}, @@ -217,7 +221,7 @@ def test_websocket(test_config: FaradayTestConfig, tmp_config): ] }, { # 7 - "data": {"action": "RUN", "agent_id": 1, "args": ["out bad_json"]}, + "data": {"action": "RUN", "agent_id": 1, "args": {"out": "bad_json"}}, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "ERROR", @@ -227,7 +231,7 @@ def test_websocket(test_config: FaradayTestConfig, tmp_config): ] }, { # 8 - "data": {"action": "RUN", "agent_id": 1, "args": ["out str"]}, + "data": {"action": "RUN", "agent_id": 1, "args": {"out": "str"}}, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "ERROR", "msg": "JSON Parsing error: Expecting value"}, @@ -235,7 +239,7 @@ def test_websocket(test_config: FaradayTestConfig, tmp_config): ] }, { # 9 - "data": {"action": "RUN", "agent_id": 1, "args": ["out none", "err"]}, + "data": {"action": "RUN", "agent_id": 1, "args": {"out": "none", "err": "T"}}, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "DEBUG", "msg": "Print by stderr"}, @@ -243,14 +247,14 @@ def test_websocket(test_config: FaradayTestConfig, tmp_config): ] }, { # 10 - "data": {"action": "RUN", "agent_id": 1, "args": ["out none", "fails"]}, + "data": {"action": "RUN", "agent_id": 1, "args": {"out": "none", "fails": "T"}}, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "WARNING", "msg": "Executor finished with exit code 1"}, ] }, { # 11 - "data": {"action": "RUN", "agent_id": 1, "args": ["out none", "err", "fails"]}, + "data": {"action": "RUN", "agent_id": 1, "args": {"out": "none", "err": "T", "fails": "T"}}, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "DEBUG", "msg": "Print by stderr"}, @@ -258,17 +262,17 @@ def test_websocket(test_config: FaradayTestConfig, tmp_config): ] }, { # 12 - "data": {"action": "RUN", "agent_id": 1, "args": ["out json"]}, + "data": {"action": "RUN", "agent_id": 1, "args": {"out": "json"}}, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "INFO", "msg": "Data sent to bulk create", "max_count": 0, "min_count": 0}, {"levelname": "INFO", "msg": "Executor finished successfully"} ], - "varenvs": {"DO_NOTHING": "True"} + "varenvs": {"DO_NOTHING": "True"}, }, { # 13 - "data": {"action": "RUN", "agent_id": 1, "args": ["err", "fails"]}, + "data": {"action": "RUN", "agent_id": 1, "args": {"err": "T", "fails": "T"}}, "logs": [ {"levelname": "INFO", "msg": "Running executor", "max_count": 0, "min_count": 0}, @@ -276,7 +280,7 @@ def test_websocket(test_config: FaradayTestConfig, tmp_config): ] }, { # 14 - "data": {"action": "RUN", "agent_id": 1, "args": ["out json", "WTF"]}, + "data": {"action": "RUN", "agent_id": 1, "args": {"out": "json", "WTF": "T"}}, "logs": [ {"levelname": "INFO", "msg": "Running executor", "max_count": 0, "min_count": 0}, @@ -288,8 +292,7 @@ def test_websocket(test_config: FaradayTestConfig, tmp_config): ] }, { - "data": {"action": "RUN", "agent_id": 1}, - "args": ["out json"], + "data": {"action": "RUN", "agent_id": 1, "args": {"out": "json"}}, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "ERROR", @@ -299,8 +302,7 @@ def test_websocket(test_config: FaradayTestConfig, tmp_config): "workspace": "error500" }, { - "data": {"action": "RUN", "agent_id": 1}, - "args": ["out json"], + "data": {"action": "RUN", "agent_id": 1, "args": {"out": "json"}}, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "ERROR", @@ -310,12 +312,13 @@ def test_websocket(test_config: FaradayTestConfig, tmp_config): "workspace": "error429" }, ]) -async def test_run_once(test_config: FaradayTestConfig, tmp_default_config, test_logger_handler, executor_options): +async def test_run_once(test_config: FaradayTestConfig, tmp_default_config, test_logger_handler, + test_logger_folder, executor_options): # Config workspace = test_config.workspace if "workspace" not in executor_options else executor_options["workspace"] configuration.set(Sections.SERVER, "api_port", str(test_config.client.port)) configuration.set(Sections.SERVER, "host", test_config.client.host) - configuration.set(Sections.SERVER, "workspace", test_config.workspace) + configuration.set(Sections.SERVER, "workspace", workspace) configuration.set(Sections.TOKENS, "registration", test_config.registration_token) configuration.set(Sections.TOKENS, "agent", test_config.agent_token) path_to_basic_executor = ( diff --git a/tests/utils/testing_faraday_server.py b/tests/utils/testing_faraday_server.py index 867e970b..12a69719 100644 --- a/tests/utils/testing_faraday_server.py +++ b/tests/utils/testing_faraday_server.py @@ -9,7 +9,7 @@ from itsdangerous import TimestampSigner import logging from logging import StreamHandler -from faraday_agent_dispatcher.logger import get_logger +from faraday_agent_dispatcher.logger import get_logger, reset_logger from queue import Queue from faraday_agent_dispatcher.config import ( @@ -173,7 +173,12 @@ def emit(self, record): self.history.append(record) -@pytest.fixture +@pytest.fixture(scope="session") +def test_logger_folder(): + reset_logger("./logs") + + +@pytest.fixture() def test_logger_handler(): logger_handler = TestLoggerHandler() logger = get_logger() From 5a69748f6de4d36bf69e4751ef830f7f296028af Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Thu, 7 Nov 2019 14:28:05 -0300 Subject: [PATCH 06/41] [ADD] websocket responses and connect tests idea --- faraday_agent_dispatcher/dispatcher.py | 31 ++++--- tests/unittests/test_agent_dispatcher.py | 100 ++++++++++++++++++++++- 2 files changed, 115 insertions(+), 16 deletions(-) diff --git a/faraday_agent_dispatcher/dispatcher.py b/faraday_agent_dispatcher/dispatcher.py index f47e9efb..d92ee9d0 100644 --- a/faraday_agent_dispatcher/dispatcher.py +++ b/faraday_agent_dispatcher/dispatcher.py @@ -112,22 +112,28 @@ async def register(self): self.websocket_token = await self.reset_websocket_token() - async def connect(self): + async def connect(self, out_func=None): if not self.websocket_token: return - async with websockets.connect(websocket_url(self.host, self.websocket_port)) as websocket: - await websocket.send(json.dumps({ - 'action': 'JOIN_AGENT', - 'workspace': self.workspace, - 'token': self.websocket_token, - })) + connected_data = json.dumps({ + 'action': 'JOIN_AGENT', + 'workspace': self.workspace, + 'token': self.websocket_token, + }) - logger.info("Connection to Faraday server succeeded") - self.websocket = websocket + if out_func is None: - await self.run_await() # This line can we called from outside (in main) + async with websockets.connect(websocket_url(self.host, self.websocket_port)) as websocket: + await websocket.send(connected_data) + + logger.info("Connection to Faraday server succeeded") + self.websocket = websocket + + await self.run_await() # This line can we called from outside (in main) + else: + await out_func(connected_data) async def run_await(self): while True: @@ -135,7 +141,8 @@ async def run_await(self): data = await self.websocket.recv() asyncio.create_task(self.run_once(data)) - async def run_once(self, data:str= None): + async def run_once(self, data:str= None, out_f=None): + out_f = out_f if out_f is not None else self.websocket.send logger.info('Parsing data: %s', data) data_dict = json.loads(data) if "action" in data_dict: @@ -183,9 +190,9 @@ async def run_once(self, data:str= None): f"Executor finished with exit code {process.returncode}") else: logger.info("Action unrecognized") - else: logger.info("Data not contains action to do") + await out_f("INVALID COMMAND") async def create_process(self, args): env = os.environ.copy() diff --git a/tests/unittests/test_agent_dispatcher.py b/tests/unittests/test_agent_dispatcher.py index d7bb5ffd..1bbbf8fe 100644 --- a/tests/unittests/test_agent_dispatcher.py +++ b/tests/unittests/test_agent_dispatcher.py @@ -151,12 +151,18 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "data": {"agent_id": 1}, "logs": [ {"levelname": "INFO", "msg": "Data not contains action to do"}, + ], + "ws_responses": [ + {"error": "'action' key is mandatory in this websocket connection"} ] }, { # 1 "data": {"action": "CUT", "agent_id": 1}, "logs": [ {"levelname": "INFO", "msg": "Action unrecognized"}, + ], + "ws_responses": [ + {"error": "Unrecognized action"} ] }, { # 2 @@ -165,6 +171,11 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "INFO", "msg": "Data sent to bulk create"}, {"levelname": "INFO", "msg": "Executor finished successfully"} + ], + "ws_responses": [ + {"action": "RUN_STATUS", + "successful": True, + "message": "Running executor"} ] }, { # 3 @@ -173,6 +184,11 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "ERROR", "msg": "JSON Parsing error: Extra data"}, {"levelname": "INFO", "msg": "Executor finished successfully"} + ], + "ws_responses": [ + {"action": "RUN_STATUS", + "successful": True, + "message": "Running executor"} ] }, { # 4 @@ -182,6 +198,11 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "INFO", "msg": "Data sent to bulk create", "min_count": 5}, {"levelname": "INFO", "msg": "Executor finished successfully"} + ], + "ws_responses": [ + {"action": "RUN_STATUS", + "successful": True, + "message": "Running executor"} ] }, { # 5 @@ -189,6 +210,11 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "INFO", "msg": "Executor finished successfully"} + ], + "ws_responses": [ + {"action": "RUN_STATUS", + "successful": True, + "message": "Running executor"} ] }, { # 6 @@ -198,6 +224,11 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "INFO", "msg": "Data sent to bulk create", "max_count": 1}, {"levelname": "INFO", "msg": "Executor finished successfully"} + ], + "ws_responses": [ + {"action": "RUN_STATUS", + "successful": True, + "message": "Running executor"} ] }, { # 7 @@ -208,6 +239,11 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "msg": "Invalid data supplied by the executor to the bulk create endpoint. " "Server responded: "}, {"levelname": "INFO", "msg": "Executor finished successfully"} + ], + "ws_responses": [ + {"action": "RUN_STATUS", + "successful": True, + "message": "Running executor"} ] }, { # 8 @@ -216,6 +252,11 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "ERROR", "msg": "JSON Parsing error: Expecting value"}, {"levelname": "INFO", "msg": "Executor finished successfully"} + ], + "ws_responses": [ + {"action": "RUN_STATUS", + "successful": True, + "message": "Running executor"} ] }, { # 9 @@ -224,6 +265,11 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "DEBUG", "msg": "Print by stderr"}, {"levelname": "INFO", "msg": "Executor finished successfully"} + ], + "ws_responses": [ + {"action": "RUN_STATUS", + "successful": True, + "message": "Running executor"} ] }, { # 10 @@ -231,6 +277,11 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "WARNING", "msg": "Executor finished with exit code 1"}, + ], + "ws_responses": [ + {"action": "RUN_STATUS", + "successful": True, + "message": "Running executor"} ] }, { # 11 @@ -239,6 +290,11 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "DEBUG", "msg": "Print by stderr"}, {"levelname": "WARNING", "msg": "Executor finished with exit code 1"}, + ], + "ws_responses": [ + {"action": "RUN_STATUS", + "successful": True, + "message": "Running executor"} ] }, { # 12 @@ -257,6 +313,11 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default {"levelname": "INFO", "msg": "Running executor", "max_count": 0, "min_count": 0}, {"levelname": "ERROR", "msg": "Mandatory argument not passed"}, + ], + "ws_responses": [ + {"action": "RUN_STATUS", + "successful": True, + "message": "Running executor"} ] }, { # 14 @@ -269,6 +330,11 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default {"levelname": "INFO", "msg": "Executor finished successfully", "max_count": 0, "min_count": 0}, {"levelname": "ERROR", "msg": "Unexpected argument passed"}, + ], + "ws_responses": [ + {"action": "RUN_STATUS", + "successful": True, + "message": "Running executor"} ] }, { @@ -279,7 +345,12 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "msg": "Invalid data supplied by the executor to the bulk create endpoint. Server responded: "}, {"levelname": "INFO", "msg": "Executor finished successfully"} ], - "workspace": "error500" + "workspace": "error500", + "ws_responses": [ + {"action": "RUN_STATUS", + "successful": True, + "message": "Running executor"} + ] }, { "data": {"action": "RUN", "agent_id": 1, "args": {"out": "json"}}, @@ -289,7 +360,12 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "msg": "Invalid data supplied by the executor to the bulk create endpoint. Server responded: "}, {"levelname": "INFO", "msg": "Executor finished successfully"} ], - "workspace": "error429" + "workspace": "error429", + "ws_responses": [ + {"action": "RUN_STATUS", + "successful": True, + "message": "Running executor"} + ] }, { "data": {"action": "RUN", "agent_id": 1, "args": {"out": "json"}}, @@ -299,7 +375,12 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "bigger limiting size in config"}, {"levelname": "INFO", "msg": "Executor finished successfully"} ], - "max_size": "1" + "max_size": "1", + "ws_responses": [ + {"action": "RUN_STATUS", + "successful": True, + "message": "Running executor"} + ] }, ]) async def test_run_once(test_config: FaradayTestConfig, tmp_default_config, test_logger_handler, @@ -328,13 +409,24 @@ async def test_run_once(test_config: FaradayTestConfig, tmp_default_config, test tmp_default_config.save() + async def eq(msg): + import json + msg_ = json.loads(msg) + assert msg_ in executor_options["ws_responses"] + executor_options["ws_responses"].remove(msg_) + # Init and register it dispatcher = Dispatcher(test_config.client.session, tmp_default_config.config_file_path) - await dispatcher.run_once(json.dumps(executor_options["data"])) + await dispatcher.run_once(json.dumps(executor_options["data"]), eq) history = test_logger_handler.history + assert len(executor_options["ws_responses"]) == 0 for l in executor_options["logs"]: min_count = 1 if "min_count" not in l else l["min_count"] max_count = sys.maxsize if "max_count" not in l else l["max_count"] assert max_count >= \ len(list(filter(lambda x: x.levelname == l["levelname"] and l["msg"] in x.message, history))) >= \ min_count, l["msg"] + + +async def test_connect(): + pass # TODO From a6e5f4ee1a726ece62759b8103b0ea56f64d749c Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Thu, 7 Nov 2019 15:42:04 -0300 Subject: [PATCH 07/41] [ADD] Unexpected ws actions and data, now acts with proper response --- faraday_agent_dispatcher/dispatcher.py | 5 +++-- tests/unittests/test_agent_dispatcher.py | 16 +++++++++++----- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/faraday_agent_dispatcher/dispatcher.py b/faraday_agent_dispatcher/dispatcher.py index d92ee9d0..7a858368 100644 --- a/faraday_agent_dispatcher/dispatcher.py +++ b/faraday_agent_dispatcher/dispatcher.py @@ -189,10 +189,11 @@ async def run_once(self, data:str= None, out_f=None): logger.warning( f"Executor finished with exit code {process.returncode}") else: - logger.info("Action unrecognized") + logger.info("Unrecognized action") + await out_f(json.dumps({f"{data_dict['action']}_RESPONSE": "Error: Unrecognized action"})) else: logger.info("Data not contains action to do") - await out_f("INVALID COMMAND") + await out_f(json.dumps({"error": "'action' key is mandatory in this websocket connection"})) async def create_process(self, args): env = os.environ.copy() diff --git a/tests/unittests/test_agent_dispatcher.py b/tests/unittests/test_agent_dispatcher.py index 1bbbf8fe..ed1af6b5 100644 --- a/tests/unittests/test_agent_dispatcher.py +++ b/tests/unittests/test_agent_dispatcher.py @@ -159,10 +159,10 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default { # 1 "data": {"action": "CUT", "agent_id": 1}, "logs": [ - {"levelname": "INFO", "msg": "Action unrecognized"}, + {"levelname": "INFO", "msg": "Unrecognized action"}, ], "ws_responses": [ - {"error": "Unrecognized action"} + {"CUT_RESPONSE": "Error: Unrecognized action"} ] }, { # 2 @@ -285,7 +285,11 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default ] }, { # 11 - "data": {"action": "RUN", "agent_id": 1, "args": {"out": "none", "err": "T", "fails": "T"}}, + "data": { + "action": "RUN", + "agent_id": 1, + "args": {"out": "none", "err": "T", "fails": "T"} + }, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "DEBUG", "msg": "Print by stderr"}, @@ -342,7 +346,8 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "ERROR", - "msg": "Invalid data supplied by the executor to the bulk create endpoint. Server responded: "}, + "msg": "Invalid data supplied by the executor to the bulk create endpoint. " + "Server responded: "}, {"levelname": "INFO", "msg": "Executor finished successfully"} ], "workspace": "error500", @@ -357,7 +362,8 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "ERROR", - "msg": "Invalid data supplied by the executor to the bulk create endpoint. Server responded: "}, + "msg": "Invalid data supplied by the executor to the bulk create endpoint. " + "Server responded: "}, {"levelname": "INFO", "msg": "Executor finished successfully"} ], "workspace": "error429", From 26cdf220ef2479d91f774611dd6666b1c55e3ce3 Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Thu, 7 Nov 2019 16:40:27 -0300 Subject: [PATCH 08/41] [ADD] Messages differ on running/success --- faraday_agent_dispatcher/dispatcher.py | 36 ++++- tests/unittests/test_agent_dispatcher.py | 172 +++++++++++++++++------ 2 files changed, 166 insertions(+), 42 deletions(-) diff --git a/faraday_agent_dispatcher/dispatcher.py b/faraday_agent_dispatcher/dispatcher.py index 7a858368..8b93bedd 100644 --- a/faraday_agent_dispatcher/dispatcher.py +++ b/faraday_agent_dispatcher/dispatcher.py @@ -161,6 +161,13 @@ async def run_once(self, data:str= None, out_f=None): ]) if not all_accepted: logger.error("Unexpected argument passed") + await out_f( + json.dumps({ + "action": "RUN_STATUS", + "running": False, + "message": f"Running executor from agent {self.agent_name}" + "Mandatory argument not passed" + }) + ) mandatory_full = all( [ config.get(Sections.PARAMS, param) != "True" # All params is not mandatory @@ -172,22 +179,49 @@ async def run_once(self, data:str= None, out_f=None): ) if not mandatory_full: logger.error("Mandatory argument not passed") + await out_f( + json.dumps({ + "action": "RUN_STATUS", + "running": False, + "message": f"Running executor from agent {self.agent_name}" + "Mandatory argument not passed" + }) + ) if mandatory_full and all_accepted: + running_msg = f"Running executor from agent {self.agent_name}" logger.info('Running executor') + process = await self.create_process(passed_params) tasks = [StdOutLineProcessor(process, self.session).process_f(), StdErrLineProcessor(process).process_f(), ] - + await out_f( + json.dumps({ + "action": "RUN_STATUS", + "running": True, + "message": running_msg + }) + ) await asyncio.gather(*tasks) await process.communicate() assert process.returncode is not None if process.returncode == 0: logger.info("Executor finished successfully") + await out_f( + json.dumps({ + "action": "RUN_STATUS", + "successful": True, + "message": "Executor finished successfully" + })) else: logger.warning( f"Executor finished with exit code {process.returncode}") + await out_f( + json.dumps({ + "action": "RUN_STATUS", + "successful": False, + "message": "Executor failed" + })) else: logger.info("Unrecognized action") await out_f(json.dumps({f"{data_dict['action']}_RESPONSE": "Error: Unrecognized action"})) diff --git a/tests/unittests/test_agent_dispatcher.py b/tests/unittests/test_agent_dispatcher.py index ed1af6b5..9ef36958 100644 --- a/tests/unittests/test_agent_dispatcher.py +++ b/tests/unittests/test_agent_dispatcher.py @@ -173,9 +173,15 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default {"levelname": "INFO", "msg": "Executor finished successfully"} ], "ws_responses": [ - {"action": "RUN_STATUS", - "successful": True, - "message": "Running executor"} + { + "action": "RUN_STATUS", + "running": True, + "message": "Running executor from agent unnamed_agent" + }, { + "action": "RUN_STATUS", + "successful": True, + "message": "Executor finished successfully" + } ] }, { # 3 @@ -186,9 +192,15 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default {"levelname": "INFO", "msg": "Executor finished successfully"} ], "ws_responses": [ - {"action": "RUN_STATUS", - "successful": True, - "message": "Running executor"} + { + "action": "RUN_STATUS", + "running": True, + "message": "Running executor from agent unnamed_agent" + }, { + "action": "RUN_STATUS", + "successful": True, + "message": "Executor finished successfully" + } ] }, { # 4 @@ -200,21 +212,37 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default {"levelname": "INFO", "msg": "Executor finished successfully"} ], "ws_responses": [ - {"action": "RUN_STATUS", - "successful": True, - "message": "Running executor"} + { + "action": "RUN_STATUS", + "running": True, + "message": "Running executor from agent unnamed_agent" + }, { + "action": "RUN_STATUS", + "successful": True, + "message": "Executor finished successfully" + } ] }, { # 5 - "data": {"action": "RUN", "agent_id": 1, "args": {"out": "json", "spaced_before":"T"}}, + "data": { + "action": "RUN", + "agent_id": 1, + "args": {"out": "json", "spaced_before": "T"} + }, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "INFO", "msg": "Executor finished successfully"} ], "ws_responses": [ - {"action": "RUN_STATUS", - "successful": True, - "message": "Running executor"} + { + "action": "RUN_STATUS", + "running": True, + "message": "Running executor from agent unnamed_agent" + }, { + "action": "RUN_STATUS", + "successful": True, + "message": "Executor finished successfully" + } ] }, { # 6 @@ -226,9 +254,15 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default {"levelname": "INFO", "msg": "Executor finished successfully"} ], "ws_responses": [ - {"action": "RUN_STATUS", - "successful": True, - "message": "Running executor"} + { + "action": "RUN_STATUS", + "running": True, + "message": "Running executor from agent unnamed_agent" + }, { + "action": "RUN_STATUS", + "successful": True, + "message": "Executor finished successfully" + } ] }, { # 7 @@ -241,9 +275,15 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default {"levelname": "INFO", "msg": "Executor finished successfully"} ], "ws_responses": [ - {"action": "RUN_STATUS", - "successful": True, - "message": "Running executor"} + { + "action": "RUN_STATUS", + "running": True, + "message": "Running executor from agent unnamed_agent" + }, { + "action": "RUN_STATUS", + "successful": True, + "message": "Executor finished successfully" + } ] }, { # 8 @@ -254,9 +294,15 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default {"levelname": "INFO", "msg": "Executor finished successfully"} ], "ws_responses": [ - {"action": "RUN_STATUS", - "successful": True, - "message": "Running executor"} + { + "action": "RUN_STATUS", + "running": True, + "message": "Running executor from agent unnamed_agent" + }, { + "action": "RUN_STATUS", + "successful": True, + "message": "Executor finished successfully" + } ] }, { # 9 @@ -267,9 +313,15 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default {"levelname": "INFO", "msg": "Executor finished successfully"} ], "ws_responses": [ - {"action": "RUN_STATUS", - "successful": True, - "message": "Running executor"} + { + "action": "RUN_STATUS", + "running": True, + "message": "Running executor from agent unnamed_agent" + }, { + "action": "RUN_STATUS", + "successful": True, + "message": "Executor finished successfully" + } ] }, { # 10 @@ -279,9 +331,15 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default {"levelname": "WARNING", "msg": "Executor finished with exit code 1"}, ], "ws_responses": [ - {"action": "RUN_STATUS", - "successful": True, - "message": "Running executor"} + { + "action": "RUN_STATUS", + "running": True, + "message": "Running executor from agent unnamed_agent" + }, { + "action": "RUN_STATUS", + "successful": False, + "message": "Executor failed" + } ] }, { # 11 @@ -296,9 +354,15 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default {"levelname": "WARNING", "msg": "Executor finished with exit code 1"}, ], "ws_responses": [ - {"action": "RUN_STATUS", - "successful": True, - "message": "Running executor"} + { + "action": "RUN_STATUS", + "running": True, + "message": "Running executor from agent unnamed_agent" + }, { + "action": "RUN_STATUS", + "successful": False, + "message": "Executor failed" + } ] }, { # 12 @@ -310,6 +374,17 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default {"levelname": "INFO", "msg": "Executor finished successfully"} ], "varenvs": {"DO_NOTHING": "True"}, + "ws_responses": [ + { + "action": "RUN_STATUS", + "running": True, + "message": "Running executor from agent unnamed_agent" + }, { + "action": "RUN_STATUS", + "successful": True, + "message": "Executor finished successfully" + } + ] }, { # 13 "data": {"action": "RUN", "agent_id": 1, "args": {"err": "T", "fails": "T"}}, @@ -352,9 +427,15 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default ], "workspace": "error500", "ws_responses": [ - {"action": "RUN_STATUS", - "successful": True, - "message": "Running executor"} + { + "action": "RUN_STATUS", + "running": True, + "message": "Running executor from agent unnamed_agent" + }, { + "action": "RUN_STATUS", + "successful": True, + "message": "Executor finished successfully" + } ] }, { @@ -368,9 +449,15 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default ], "workspace": "error429", "ws_responses": [ - {"action": "RUN_STATUS", - "successful": True, - "message": "Running executor"} + { + "action": "RUN_STATUS", + "running": True, + "message": "Running executor from agent unnamed_agent" + }, { + "action": "RUN_STATUS", + "successful": True, + "message": "Executor finished successfully" + } ] }, { @@ -379,13 +466,16 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "ERROR", "msg": "ValueError raised processing stdout, try with " "bigger limiting size in config"}, - {"levelname": "INFO", "msg": "Executor finished successfully"} + {"levelname": "INFO", "msg": "Executor finished successfully", "max_count": 0, + "min_count": 0} ], "max_size": "1", "ws_responses": [ - {"action": "RUN_STATUS", - "successful": True, - "message": "Running executor"} + { + "action": "RUN_STATUS", + "running": True, + "message": "Running executor from agent unnamed_agent" + } ] }, ]) From 207b148a8175b28ab56d33a4a8bf7cd8ac190dfc Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Thu, 7 Nov 2019 17:46:20 -0300 Subject: [PATCH 09/41] [ADD] Rest of tests with ws messages --- faraday_agent_dispatcher/dispatcher.py | 6 +-- tests/unittests/test_agent_dispatcher.py | 58 +++++++++++++----------- 2 files changed, 34 insertions(+), 30 deletions(-) diff --git a/faraday_agent_dispatcher/dispatcher.py b/faraday_agent_dispatcher/dispatcher.py index 8b93bedd..5ca06410 100644 --- a/faraday_agent_dispatcher/dispatcher.py +++ b/faraday_agent_dispatcher/dispatcher.py @@ -165,7 +165,7 @@ async def run_once(self, data:str= None, out_f=None): json.dumps({ "action": "RUN_STATUS", "running": False, - "message": f"Running executor from agent {self.agent_name}" + "Mandatory argument not passed" + "message": f"Unexpected argument(s) passed to {self.agent_name} agent" }) ) mandatory_full = all( @@ -183,12 +183,12 @@ async def run_once(self, data:str= None, out_f=None): json.dumps({ "action": "RUN_STATUS", "running": False, - "message": f"Running executor from agent {self.agent_name}" + "Mandatory argument not passed" + "message": f"Mandatory argument(s) not passed to {self.agent_name} agent" }) ) if mandatory_full and all_accepted: - running_msg = f"Running executor from agent {self.agent_name}" + running_msg = f"Running executor from {self.agent_name} agent" logger.info('Running executor') process = await self.create_process(passed_params) diff --git a/tests/unittests/test_agent_dispatcher.py b/tests/unittests/test_agent_dispatcher.py index 9ef36958..d7f3a223 100644 --- a/tests/unittests/test_agent_dispatcher.py +++ b/tests/unittests/test_agent_dispatcher.py @@ -176,7 +176,7 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default { "action": "RUN_STATUS", "running": True, - "message": "Running executor from agent unnamed_agent" + "message": "Running executor from unnamed_agent agent" }, { "action": "RUN_STATUS", "successful": True, @@ -195,7 +195,7 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default { "action": "RUN_STATUS", "running": True, - "message": "Running executor from agent unnamed_agent" + "message": "Running executor from unnamed_agent agent" }, { "action": "RUN_STATUS", "successful": True, @@ -215,7 +215,7 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default { "action": "RUN_STATUS", "running": True, - "message": "Running executor from agent unnamed_agent" + "message": "Running executor from unnamed_agent agent" }, { "action": "RUN_STATUS", "successful": True, @@ -237,7 +237,7 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default { "action": "RUN_STATUS", "running": True, - "message": "Running executor from agent unnamed_agent" + "message": "Running executor from unnamed_agent agent" }, { "action": "RUN_STATUS", "successful": True, @@ -257,7 +257,7 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default { "action": "RUN_STATUS", "running": True, - "message": "Running executor from agent unnamed_agent" + "message": "Running executor from unnamed_agent agent" }, { "action": "RUN_STATUS", "successful": True, @@ -278,7 +278,7 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default { "action": "RUN_STATUS", "running": True, - "message": "Running executor from agent unnamed_agent" + "message": "Running executor from unnamed_agent agent" }, { "action": "RUN_STATUS", "successful": True, @@ -297,7 +297,7 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default { "action": "RUN_STATUS", "running": True, - "message": "Running executor from agent unnamed_agent" + "message": "Running executor from unnamed_agent agent" }, { "action": "RUN_STATUS", "successful": True, @@ -316,7 +316,7 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default { "action": "RUN_STATUS", "running": True, - "message": "Running executor from agent unnamed_agent" + "message": "Running executor from unnamed_agent agent" }, { "action": "RUN_STATUS", "successful": True, @@ -334,7 +334,7 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default { "action": "RUN_STATUS", "running": True, - "message": "Running executor from agent unnamed_agent" + "message": "Running executor from unnamed_agent agent" }, { "action": "RUN_STATUS", "successful": False, @@ -357,7 +357,7 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default { "action": "RUN_STATUS", "running": True, - "message": "Running executor from agent unnamed_agent" + "message": "Running executor from unnamed_agent agent" }, { "action": "RUN_STATUS", "successful": False, @@ -378,7 +378,7 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default { "action": "RUN_STATUS", "running": True, - "message": "Running executor from agent unnamed_agent" + "message": "Running executor from unnamed_agent agent" }, { "action": "RUN_STATUS", "successful": True, @@ -394,9 +394,11 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default {"levelname": "ERROR", "msg": "Mandatory argument not passed"}, ], "ws_responses": [ - {"action": "RUN_STATUS", - "successful": True, - "message": "Running executor"} + { + "action": "RUN_STATUS", + "running": False, + "message": "Mandatory argument(s) not passed to unnamed_agent agent" + } ] }, { # 14 @@ -412,11 +414,11 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default ], "ws_responses": [ {"action": "RUN_STATUS", - "successful": True, - "message": "Running executor"} + "running": False, + "message": "Unexpected argument(s) passed to unnamed_agent agent"} ] }, - { + { # 15 "data": {"action": "RUN", "agent_id": 1, "args": {"out": "json"}}, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, @@ -430,7 +432,7 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default { "action": "RUN_STATUS", "running": True, - "message": "Running executor from agent unnamed_agent" + "message": "Running executor from unnamed_agent agent" }, { "action": "RUN_STATUS", "successful": True, @@ -438,7 +440,7 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default } ] }, - { + { # 16 "data": {"action": "RUN", "agent_id": 1, "args": {"out": "json"}}, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, @@ -452,7 +454,7 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default { "action": "RUN_STATUS", "running": True, - "message": "Running executor from agent unnamed_agent" + "message": "Running executor from unnamed_agent agent" }, { "action": "RUN_STATUS", "successful": True, @@ -460,21 +462,24 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default } ] }, - { + { # 17 "data": {"action": "RUN", "agent_id": 1, "args": {"out": "json"}}, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "ERROR", "msg": "ValueError raised processing stdout, try with " "bigger limiting size in config"}, - {"levelname": "INFO", "msg": "Executor finished successfully", "max_count": 0, - "min_count": 0} + {"levelname": "INFO", "msg": "Executor finished successfully"} ], "max_size": "1", "ws_responses": [ { "action": "RUN_STATUS", "running": True, - "message": "Running executor from agent unnamed_agent" + "message": "Running executor from unnamed_agent agent" + }, { + "action": "RUN_STATUS", + "successful": True, + "message": "Executor finished successfully" } ] }, @@ -505,15 +510,14 @@ async def test_run_once(test_config: FaradayTestConfig, tmp_default_config, test tmp_default_config.save() - async def eq(msg): - import json + async def ws_messages_checker(msg): msg_ = json.loads(msg) assert msg_ in executor_options["ws_responses"] executor_options["ws_responses"].remove(msg_) # Init and register it dispatcher = Dispatcher(test_config.client.session, tmp_default_config.config_file_path) - await dispatcher.run_once(json.dumps(executor_options["data"]), eq) + await dispatcher.run_once(json.dumps(executor_options["data"]), ws_messages_checker) history = test_logger_handler.history assert len(executor_options["ws_responses"]) == 0 for l in executor_options["logs"]: From 8259fff561123f055031cd8d7f38bebcdc8a2dc0 Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Thu, 7 Nov 2019 18:10:43 -0300 Subject: [PATCH 10/41] [MOD] Multiple executors config ini file, and run_once tests --- tests/data/test_config.ini | 11 +- tests/unittests/test_agent_dispatcher.py | 125 ++++++++++++++++++++--- 2 files changed, 117 insertions(+), 19 deletions(-) diff --git a/tests/data/test_config.ini b/tests/data/test_config.ini index 2431bce0..568b42eb 100644 --- a/tests/data/test_config.ini +++ b/tests/data/test_config.ini @@ -4,13 +4,16 @@ host = localhost api_port = 5985 websocket_port = 9000 -[executor] -; Complete the cmd option with the command you want the dispatcher to run -cmd = exit 1 +[agent] agent_name = unnamed_agent -max_size = 65536 +executors = [ex1] [tokens] ; To get your registration token, visit http://localhost:5985/#/admin/agents, copy ; the value and uncomment the line registration = 1234567890123456789012345 + +[ex1] +; Complete the cmd option with the command you want the dispatcher to run +cmd = exit 1 +max_size = 65536 \ No newline at end of file diff --git a/tests/unittests/test_agent_dispatcher.py b/tests/unittests/test_agent_dispatcher.py index d7f3a223..6fc400fb 100644 --- a/tests/unittests/test_agent_dispatcher.py +++ b/tests/unittests/test_agent_dispatcher.py @@ -184,8 +184,30 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default } ] }, + { # 2 + "data": {"action": "RUN", "agent_id": 1, "executor": "ex1", "args": {"out": "json"}}, + "logs": [ + {"levelname": "INFO", "msg": "Running executor"}, + {"levelname": "INFO", "msg": "Data sent to bulk create"}, + {"levelname": "INFO", "msg": "Executor finished successfully"} + ], + "ws_responses": [ + { + "action": "RUN_STATUS", + "running": True, + "message": "Running executor from unnamed_agent agent" + }, { + "action": "RUN_STATUS", + "successful": True, + "message": "Executor finished successfully" + } + ] + }, { # 3 - "data": {"action": "RUN", "agent_id": 1, "args": {"out": "json", "count": "5"}}, + "data": { + "action": "RUN", "agent_id": 1, "executor": "ex1", + "args": {"out": "json", "count": "5"} + }, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "ERROR", "msg": "JSON Parsing error: Extra data"}, @@ -204,8 +226,10 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default ] }, { # 4 - "data": {"action": "RUN", "agent_id": 1, "args": {"out": "json", "count": "5", - "spare": "T"}}, + "data": { + "action": "RUN", "agent_id": 1, "executor": "ex1", + "args": {"out": "json", "count": "5", "spare": "T"} + }, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "INFO", "msg": "Data sent to bulk create", "min_count": 5}, @@ -227,6 +251,7 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "data": { "action": "RUN", "agent_id": 1, + "executor": "ex1", "args": {"out": "json", "spaced_before": "T"} }, "logs": [ @@ -246,8 +271,10 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default ] }, { # 6 - "data": {"action": "RUN", "agent_id": 1, "args": {"out": "json", "spaced_middle": "T", - "count": "5", "spare": "T"}}, + "data": { + "action": "RUN", "agent_id": 1, "executor": "ex1", + "args": {"out": "json", "spaced_middle": "T", "count": "5", "spare": "T"} + }, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "INFO", "msg": "Data sent to bulk create", "max_count": 1}, @@ -266,7 +293,9 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default ] }, { # 7 - "data": {"action": "RUN", "agent_id": 1, "args": {"out": "bad_json"}}, + "data": { + "action": "RUN", "agent_id": 1, "executor": "ex1", "args": {"out": "bad_json"} + }, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "ERROR", @@ -287,7 +316,9 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default ] }, { # 8 - "data": {"action": "RUN", "agent_id": 1, "args": {"out": "str"}}, + "data": { + "action": "RUN", "agent_id": 1, "executor": "ex1", "args": {"out": "str"} + }, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "ERROR", "msg": "JSON Parsing error: Expecting value"}, @@ -306,7 +337,10 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default ] }, { # 9 - "data": {"action": "RUN", "agent_id": 1, "args": {"out": "none", "err": "T"}}, + "data": { + "action": "RUN", "agent_id": 1, "executor": "ex1", + "args": {"out": "none", "err": "T"} + }, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "DEBUG", "msg": "Print by stderr"}, @@ -325,7 +359,10 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default ] }, { # 10 - "data": {"action": "RUN", "agent_id": 1, "args": {"out": "none", "fails": "T"}}, + "data": { + "action": "RUN", "agent_id": 1, "executor": "ex1", + "args": {"out": "none", "fails": "T"} + }, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "WARNING", "msg": "Executor finished with exit code 1"}, @@ -346,6 +383,7 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "data": { "action": "RUN", "agent_id": 1, + "executor": "ex1", "args": {"out": "none", "err": "T", "fails": "T"} }, "logs": [ @@ -366,7 +404,12 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default ] }, { # 12 - "data": {"action": "RUN", "agent_id": 1, "args": {"out": "json"}}, + "data": { + "action": "RUN", + "agent_id": 1, + "executor": "ex1", + "args": {"out": "json"} + }, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "INFO", "msg": "Data sent to bulk create", "max_count": 0, @@ -387,7 +430,12 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default ] }, { # 13 - "data": {"action": "RUN", "agent_id": 1, "args": {"err": "T", "fails": "T"}}, + "data": { + "action": "RUN", + "agent_id": 1, + "executor": "ex1", + "args": {"err": "T", "fails": "T"}, + }, "logs": [ {"levelname": "INFO", "msg": "Running executor", "max_count": 0, "min_count": 0}, @@ -402,7 +450,10 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default ] }, { # 14 - "data": {"action": "RUN", "agent_id": 1, "args": {"out": "json", "WTF": "T"}}, + "data": { + "action": "RUN", "agent_id": 1, "executor": "ex1", + "args": {"out": "json", "WTF": "T"} + }, "logs": [ {"levelname": "INFO", "msg": "Running executor", "max_count": 0, "min_count": 0}, @@ -419,7 +470,9 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default ] }, { # 15 - "data": {"action": "RUN", "agent_id": 1, "args": {"out": "json"}}, + "data": { + "action": "RUN", "agent_id": 1, "executor": "ex1", "args": {"out": "json"} + }, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "ERROR", @@ -441,7 +494,9 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default ] }, { # 16 - "data": {"action": "RUN", "agent_id": 1, "args": {"out": "json"}}, + "data": { + "action": "RUN", "agent_id": 1, "executor": "ex1", "args": {"out": "json"} + }, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "ERROR", @@ -463,7 +518,7 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default ] }, { # 17 - "data": {"action": "RUN", "agent_id": 1, "args": {"out": "json"}}, + "data": {"action": "RUN", "agent_id": 1, "executor": "ex1", "args": {"out": "json"}}, "logs": [ {"levelname": "INFO", "msg": "Running executor"}, {"levelname": "ERROR", "msg": "ValueError raised processing stdout, try with " @@ -483,6 +538,46 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default } ] }, + { # 18 + "data": { + "action": "RUN", "agent_id": 1, + "args": {"out": "json", "WTF": "T"} + }, + "logs": [ + {"levelname": "INFO", "msg": "Running executor", "max_count": 0, + "min_count": 0}, + {"levelname": "INFO", "msg": "Data sent to bulk create", "max_count": 0, + "min_count": 0}, + {"levelname": "INFO", "msg": "Executor finished successfully", "max_count": 0, + "min_count": 0}, + {"levelname": "ERROR", "msg": "Unexpected argument passed"}, + ], + "ws_responses": [ + {"action": "RUN_STATUS", + "running": False, + "message": "No executor selected"} + ] + }, + { # 19 + "data": { + "action": "RUN", "agent_id": 1, "executor": "NOT_4N_CORRECT_EXECUTOR", + "args": {"out": "json", "WTF": "T"} + }, + "logs": [ + {"levelname": "INFO", "msg": "Running executor", "max_count": 0, + "min_count": 0}, + {"levelname": "INFO", "msg": "Data sent to bulk create", "max_count": 0, + "min_count": 0}, + {"levelname": "INFO", "msg": "Executor finished successfully", "max_count": 0, + "min_count": 0}, + {"levelname": "ERROR", "msg": "Unexpected argument passed"}, + ], + "ws_responses": [ + {"action": "RUN_STATUS", + "running": False, + "message": "The selected executor not exists"} + ] + }, ]) async def test_run_once(test_config: FaradayTestConfig, tmp_default_config, test_logger_handler, test_logger_folder, executor_options): From 203cdd751ad9f0ac53ede2e1f2fdc1a1ad744d54 Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Fri, 8 Nov 2019 11:24:07 -0300 Subject: [PATCH 11/41] [ADD] Mor built tests --- faraday_agent_dispatcher/config.py | 8 +- faraday_agent_dispatcher/executor.py | 3 + .../utils/control_values_utils.py | 12 ++- tests/data/test_config.ini | 2 +- tests/unittests/test_agent_dispatcher.py | 73 +++++++++++++++++-- 5 files changed, 87 insertions(+), 11 deletions(-) create mode 100644 faraday_agent_dispatcher/executor.py diff --git a/faraday_agent_dispatcher/config.py b/faraday_agent_dispatcher/config.py index 057bbfdf..ac3e5a5f 100644 --- a/faraday_agent_dispatcher/config.py +++ b/faraday_agent_dispatcher/config.py @@ -59,7 +59,7 @@ def save_config(filepath=None): class Sections: TOKENS = "tokens" SERVER = "server" - EXECUTOR = "executor" - VARENVS = "varenvs" - PARAMS = "params" - + AGENT = "agent" + EXECUTOR_VARENVS = "{}_varenvs" + EXECUTOR_PARAMS = "{}_params" + EXECUTOR_DATA = "{}" diff --git a/faraday_agent_dispatcher/executor.py b/faraday_agent_dispatcher/executor.py new file mode 100644 index 00000000..cf5becf1 --- /dev/null +++ b/faraday_agent_dispatcher/executor.py @@ -0,0 +1,3 @@ + +class Executor: + pass \ No newline at end of file diff --git a/faraday_agent_dispatcher/utils/control_values_utils.py b/faraday_agent_dispatcher/utils/control_values_utils.py index 6464c0a0..419f5934 100644 --- a/faraday_agent_dispatcher/utils/control_values_utils.py +++ b/faraday_agent_dispatcher/utils/control_values_utils.py @@ -1,4 +1,4 @@ - +import json def control_int(field_name,value): if value is None: raise ValueError(f"Trying to parse {field_name} with None value and should be an int") @@ -17,6 +17,16 @@ def control_host(field_name, value): control_str(field_name, value) +def control_list(field_name, value): + if not isinstance(value, str) or not isinstance(value.split(","),list): + raise ValueError(f"Trying to parse {field_name} with value {value} and should be a list") + + +def control_bool(field_name, value): + if value.lower() not in ["true", "false", "t", "f"]: + raise ValueError(f"Trying to parse {field_name} with value {value} and should be a bool") + + def control_registration_token(field_name, value): if value is None: raise ValueError(f'"{field_name}" option is required in the configuration ' diff --git a/tests/data/test_config.ini b/tests/data/test_config.ini index 568b42eb..adb24e2f 100644 --- a/tests/data/test_config.ini +++ b/tests/data/test_config.ini @@ -6,7 +6,7 @@ websocket_port = 9000 [agent] agent_name = unnamed_agent -executors = [ex1] +executors = ex1 [tokens] ; To get your registration token, visit http://localhost:5985/#/admin/agents, copy diff --git a/tests/unittests/test_agent_dispatcher.py b/tests/unittests/test_agent_dispatcher.py index 6fc400fb..891d715f 100644 --- a/tests/unittests/test_agent_dispatcher.py +++ b/tests/unittests/test_agent_dispatcher.py @@ -84,10 +84,22 @@ {"remove": {}, "replace": {Sections.TOKENS: { "agent": "QWE46aasdje446aasdje446aaQWE46aasdje446aasdje446aaQWE46aasdje446"}}}, - {"remove": {Sections.EXECUTOR: ["cmd"]}, + {"remove": {Sections.EXECUTOR_DATA.format("ex1"): ["cmd"]}, "replace": {}, "expected_exception": ValueError}, - {"remove": {Sections.EXECUTOR: ["agent_name"]}, + {"remove": {}, + "replace": {Sections.EXECUTOR_DATA.format("ex1"): {"max_size": "ASDASD"}}, + "expected_exception": ValueError}, + {"remove": {}, + "replace": {Sections.EXECUTOR_PARAMS.format("ex1"): {"param1": "ASDASD"}}, + "expected_exception": ValueError}, + {"remove": {}, + "replace": {Sections.EXECUTOR_PARAMS.format("ex1"): {"param1": "5"}}, + "expected_exception": ValueError}, + {"remove": {}, + "replace": {Sections.EXECUTOR_PARAMS.format("ex1"): {"param1": "True"}} + }, + {"remove": {Sections.AGENT: ["agent_name"]}, "replace": {}, "expected_exception": ValueError}, {"remove": {}, @@ -114,7 +126,7 @@ async def test_start_and_register(test_config: FaradayTestConfig, tmp_default_co configuration.set(Sections.SERVER, "host", test_config.client.host) configuration.set(Sections.SERVER, "workspace", test_config.workspace) configuration.set(Sections.TOKENS, "registration", test_config.registration_token) - configuration.set(Sections.EXECUTOR, "cmd", 'exit 1') + configuration.set(Sections.EXECUTOR_DATA, "cmd", 'exit 1') tmp_default_config.save() # Init and register it @@ -623,5 +635,56 @@ async def ws_messages_checker(msg): min_count, l["msg"] -async def test_connect(): - pass # TODO +async def test_connect(test_config: FaradayTestConfig, tmp_default_config, test_logger_handler, + test_logger_folder): + configuration.set(Sections.SERVER, "api_port", str(test_config.client.port)) + configuration.set(Sections.SERVER, "host", test_config.client.host) + configuration.set(Sections.SERVER, "workspace", test_config.workspace) + configuration.set(Sections.TOKENS, "registration", test_config.registration_token) + configuration.set(Sections.TOKENS, "agent", test_config.agent_token) + path_to_basic_executor = ( + Path(__file__).parent.parent / + 'data' / 'basic_executor.py' + ) + configuration.set(Sections.EXECUTOR, "cmd", "python {}".format(path_to_basic_executor)) + configuration.set(Sections.EXECUTOR, "executors", "[ex1,ex2,ex3]") + configuration.set(Sections.PARAMS, "out", "True") + [configuration.set(Sections.PARAMS, param, "False") for param in [ + "count", "spare", "spaced_before", "spaced_middle", "err", "fails"]] + tmp_default_config.save() + dispatcher = Dispatcher(test_config.client.session, tmp_default_config.config_file_path) + + ws_responses = [{ + 'action': 'JOIN_AGENT', + 'workspace': test_config.workspace, + 'token': None, + 'executors': [ + { + "executor_name": "ex1", + "args": { + "param1": True, + "param2": False + } + }, + { + "executor_name": "ex2", + "args": { + "param3": False, + "param4": False + } + }, + { + "executor_name": "ex3", + "args": {} + } + ] + }] + + async def ws_messages_checker(msg): + msg_ = json.loads(msg) + assert msg_ in ws_responses + ws_responses.remove(msg_) + + await dispatcher.connect(ws_messages_checker) + + assert len(ws_responses) == 0 From 6aa0a40c650f101d3afd13015efcab3017c99026 Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Fri, 8 Nov 2019 12:05:35 -0300 Subject: [PATCH 12/41] [ADD-WIP] Executor building --- faraday_agent_dispatcher/dispatcher.py | 18 ++++++++----- faraday_agent_dispatcher/executor.py | 36 +++++++++++++++++++++++++- 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/faraday_agent_dispatcher/dispatcher.py b/faraday_agent_dispatcher/dispatcher.py index 5ca06410..bb5bbce8 100644 --- a/faraday_agent_dispatcher/dispatcher.py +++ b/faraday_agent_dispatcher/dispatcher.py @@ -30,11 +30,13 @@ control_str, control_host, control_registration_token, - control_agent_token + control_agent_token, + control_list ) import faraday_agent_dispatcher.logger as logging from faraday_agent_dispatcher.config import instance as config, Sections, save_config +from faraday_agent_dispatcher.executor import Executor logger = logging.get_logger() logging.setup_logging() @@ -53,11 +55,10 @@ class Dispatcher: "registration": control_registration_token, "agent": control_agent_token }, - Sections.EXECUTOR: { - "cmd": control_str, - "agent_name": control_str + Sections.AGENT: { + "agent_name": control_str, + "executors": control_list }, - } def __init__(self, session, config_path=None): @@ -69,11 +70,14 @@ def __init__(self, session, config_path=None): self.websocket_port = config.get(Sections.SERVER, "websocket_port") self.workspace = config.get(Sections.SERVER, "workspace") self.agent_token = config[Sections.TOKENS].get("agent", None) - self.executor_cmd = config.get(Sections.EXECUTOR, "cmd") - self.agent_name = config.get(Sections.EXECUTOR, "agent_name") + self.agent_name = config.get(Sections.AGENT, "agent_name") self.session = session self.websocket = None self.websocket_token = None + self.executors = { + executor_name: + Executor(executor_name, config) for executor_name in config[Sections.AGENT].get("executors", []).split(",") + } async def reset_websocket_token(self): # I'm built so I ask for websocket token diff --git a/faraday_agent_dispatcher/executor.py b/faraday_agent_dispatcher/executor.py index cf5becf1..94879480 100644 --- a/faraday_agent_dispatcher/executor.py +++ b/faraday_agent_dispatcher/executor.py @@ -1,3 +1,37 @@ +from faraday_agent_dispatcher.config import Sections +from faraday_agent_dispatcher.utils.control_values_utils import ( + control_int, + control_str, + control_bool +) + class Executor: - pass \ No newline at end of file + __control_dict = { + Sections.EXECUTOR_DATA: { + "cmd": control_str, + "max_size": control_int + } + } + + def __init__(self, name, config): + self.control_config(name, config) + self.name = name + executor_section = Sections.EXECUTOR_DATA.format(name) + params_section = Sections.EXECUTOR_PARAMS.format(name) + varenvs_section = Sections.EXECUTOR_VARENVS.format(name) + self.cmd = config.get(executor_section, "cmd") + self.max_size = config[executor_section].get("max_size", 64 * 1024) + self.params = config[params_section] if params_section in config else [] + self.varenvs = config[varenvs_section] if varenvs_section in config else [] + + def control_config(self, name, config): + for section in self.__control_dict: + for option in self.__control_dict[section]: + value = config.get(section.format(name), option) if option in config[section.format(name)] else None + self.__control_dict[section][option](option, value) + params_section = Sections.EXECUTOR_PARAMS.format(name) + if params_section in config: + for option in config[params_section]: + value = config.get(params_section, option) + control_bool(option, value) \ No newline at end of file From a63ec72fc3f82046ab692370a2099462a219827a Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Fri, 8 Nov 2019 12:23:36 -0300 Subject: [PATCH 13/41] [ADD] Dispatcher built --- faraday_agent_dispatcher/executor.py | 6 ++++-- tests/unittests/test_agent_dispatcher.py | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/faraday_agent_dispatcher/executor.py b/faraday_agent_dispatcher/executor.py index 94879480..56b10b68 100644 --- a/faraday_agent_dispatcher/executor.py +++ b/faraday_agent_dispatcher/executor.py @@ -22,8 +22,10 @@ def __init__(self, name, config): varenvs_section = Sections.EXECUTOR_VARENVS.format(name) self.cmd = config.get(executor_section, "cmd") self.max_size = config[executor_section].get("max_size", 64 * 1024) - self.params = config[params_section] if params_section in config else [] - self.varenvs = config[varenvs_section] if varenvs_section in config else [] + self.params = dict(config[params_section]) if params_section in config else {} + self.params = {key: value.lower() in ["t", "true"] for key, value in self.params.items()} + self.varenvs = dict(config[varenvs_section]) if varenvs_section in config else {} + def control_config(self, name, config): for section in self.__control_dict: diff --git a/tests/unittests/test_agent_dispatcher.py b/tests/unittests/test_agent_dispatcher.py index 891d715f..f8342fd6 100644 --- a/tests/unittests/test_agent_dispatcher.py +++ b/tests/unittests/test_agent_dispatcher.py @@ -108,6 +108,8 @@ def test_basic_built(tmp_custom_config, config_changes_dict): for section in config_changes_dict["replace"]: for option in config_changes_dict["replace"][section]: + if section not in configuration: + configuration.add_section(section) configuration.set(section, option, config_changes_dict["replace"][section][option]) for section in config_changes_dict["remove"]: for option in config_changes_dict["remove"][section]: From ad540f284257d247525bf75087a48a44d960bc81 Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Fri, 8 Nov 2019 13:17:14 -0300 Subject: [PATCH 14/41] [ADD] Executor inner routing --- faraday_agent_dispatcher/dispatcher.py | 187 +++++++++++--------- faraday_agent_dispatcher/example_config.ini | 18 +- faraday_agent_dispatcher/executor.py | 2 +- tests/unittests/test_agent_dispatcher.py | 40 ++--- 4 files changed, 130 insertions(+), 117 deletions(-) diff --git a/faraday_agent_dispatcher/dispatcher.py b/faraday_agent_dispatcher/dispatcher.py index bb5bbce8..a855748d 100644 --- a/faraday_agent_dispatcher/dispatcher.py +++ b/faraday_agent_dispatcher/dispatcher.py @@ -145,95 +145,121 @@ async def run_await(self): data = await self.websocket.recv() asyncio.create_task(self.run_once(data)) - async def run_once(self, data:str= None, out_f=None): - out_f = out_f if out_f is not None else self.websocket.send + async def run_once(self, data:str= None, out_func=None): + out_func = out_func if out_func is not None else self.websocket.send logger.info('Parsing data: %s', data) data_dict = json.loads(data) - if "action" in data_dict: - if data_dict["action"] == "RUN": - params = config.options(Sections.PARAMS).copy() - passed_params = data_dict['args'] if 'args' in data_dict else {} - [params.remove(param) for param in config.defaults()] - - all_accepted = all( - [ - any([ + if "action" not in data_dict: + logger.info("Data not contains action to do") + await out_func(json.dumps({"error": "'action' key is mandatory in this websocket connection"})) + return + + if data_dict["action"] not in ["RUN"]: # ONLY SUPPORTED COMMAND FOR NOW + logger.info("Unrecognized action") + await out_func(json.dumps({f"{data_dict['action']}_RESPONSE": "Error: Unrecognized action"})) + return + + if data_dict["action"] == "RUN": + if "executor" not in data_dict: + logger.error("No executor selected") + await out_func( + json.dumps({ + "action": "RUN_STATUS", + "running": False, + "message": f"No executor selected to {self.agent_name} agent" + }) + ) + + if data_dict["executor"] not in self.executors: + logger.error("The selected executor not exists") + await out_func( + json.dumps({ + "action": "RUN_STATUS", + "running": False, + "message": f"The selected executor {data_dict['executor']} not exists in {self.agent_name} " + f"agent" + }) + ) + + executor = self.executors[data_dict["executor"]] + + params = list(executor.params.keys()).copy() + passed_params = data_dict['args'] if 'args' in data_dict else {} + [params.remove(param) for param in config.defaults()] + + all_accepted = all( + [ + any([ param in passed_param # Control any available param for param in params # was passed ]) - for passed_param in passed_params # For all passed params - ]) - if not all_accepted: - logger.error("Unexpected argument passed") - await out_f( - json.dumps({ - "action": "RUN_STATUS", - "running": False, - "message": f"Unexpected argument(s) passed to {self.agent_name} agent" - }) - ) - mandatory_full = all( - [ - config.get(Sections.PARAMS, param) != "True" # All params is not mandatory - or any([ + for passed_param in passed_params # For all passed params + ]) + if not all_accepted: + logger.error("Unexpected argument passed") + await out_func( + json.dumps({ + "action": "RUN_STATUS", + "running": False, + "message": f"Unexpected argument(s) passed to {self.agent_name} agent" + }) + ) + mandatory_full = all( + [ + not executor.params[param] # All params is not mandatory + or any([ param in passed_param for passed_param in passed_params # Or was passed ]) - for param in params - ] + for param in params + ] + ) + if not mandatory_full: + logger.error("Mandatory argument not passed") + await out_func( + json.dumps({ + "action": "RUN_STATUS", + "running": False, + "message": f"Mandatory argument(s) not passed to {self.agent_name} agent" + }) ) - if not mandatory_full: - logger.error("Mandatory argument not passed") - await out_f( + + if mandatory_full and all_accepted: + running_msg = f"Running executor from {self.agent_name} agent" + logger.info('Running executor') + + process = await self.create_process(executor, passed_params) + tasks = [StdOutLineProcessor(process, self.session).process_f(), + StdErrLineProcessor(process).process_f(), + ] + await out_func( + json.dumps({ + "action": "RUN_STATUS", + "running": True, + "message": running_msg + }) + ) + await asyncio.gather(*tasks) + await process.communicate() + assert process.returncode is not None + if process.returncode == 0: + logger.info("Executor finished successfully") + await out_func( json.dumps({ "action": "RUN_STATUS", - "running": False, - "message": f"Mandatory argument(s) not passed to {self.agent_name} agent" - }) - ) - - if mandatory_full and all_accepted: - running_msg = f"Running executor from {self.agent_name} agent" - logger.info('Running executor') - - process = await self.create_process(passed_params) - tasks = [StdOutLineProcessor(process, self.session).process_f(), - StdErrLineProcessor(process).process_f(), - ] - await out_f( + "successful": True, + "message": "Executor finished successfully" + })) + else: + logger.warning( + f"Executor finished with exit code {process.returncode}") + await out_func( json.dumps({ "action": "RUN_STATUS", - "running": True, - "message": running_msg - }) - ) - await asyncio.gather(*tasks) - await process.communicate() - assert process.returncode is not None - if process.returncode == 0: - logger.info("Executor finished successfully") - await out_f( - json.dumps({ - "action": "RUN_STATUS", - "successful": True, - "message": "Executor finished successfully" - })) - else: - logger.warning( - f"Executor finished with exit code {process.returncode}") - await out_f( - json.dumps({ - "action": "RUN_STATUS", - "successful": False, - "message": "Executor failed" - })) - else: - logger.info("Unrecognized action") - await out_f(json.dumps({f"{data_dict['action']}_RESPONSE": "Error: Unrecognized action"})) - else: - logger.info("Data not contains action to do") - await out_f(json.dumps({"error": "'action' key is mandatory in this websocket connection"})) + "successful": False, + "message": "Executor failed" + })) - async def create_process(self, args): + async def create_process(self, executor: Executor, args): env = os.environ.copy() if isinstance(args, dict): for k in args: @@ -241,15 +267,14 @@ async def create_process(self, args): else: logger.error("Args from data received has a not supported type") raise ValueError("Args from data received has a not supported type") - for varenv in config.options(Sections.VARENVS): - if varenv not in config.defaults(): - env[varenv.upper()] = config.get(Sections.VARENVS,varenv) + for varenv, value in executor.varenvs.items(): + env[varenv.upper()] = value process = await asyncio.create_subprocess_shell( - self.executor_cmd, + executor.cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, env=env, - limit=int(config[Sections.EXECUTOR].get("max_size", 64 * 1024)) + limit=executor.max_size # If the config is not set, use async.io default ) return process diff --git a/faraday_agent_dispatcher/example_config.ini b/faraday_agent_dispatcher/example_config.ini index 39f96c51..04682b3a 100644 --- a/faraday_agent_dispatcher/example_config.ini +++ b/faraday_agent_dispatcher/example_config.ini @@ -4,18 +4,22 @@ host = localhost api_port = 5985 websocket_port = 9000 -[executor] -; Complete the cmd option with the command you want the dispatcher to run -; cmd = +[agent] agent_name = unnamed_agent -max_size = 65536 -; 1024 * 64 +; Complete the executor option with a comma separated list of executor names +executors = ex1 [tokens] ; To get your registration token, visit http://localhost:5985/#/admin/agents, copy ; the value and uncomment the line ; registration = -[varenvs] +[ex1] +; Complete the cmd option with the command you want the dispatcher to run +; cmd = +max_size = 65536 +; 1024 * 64 + +[ex1_varenvs] -[params] \ No newline at end of file +[ex1_params] \ No newline at end of file diff --git a/faraday_agent_dispatcher/executor.py b/faraday_agent_dispatcher/executor.py index 56b10b68..02726b11 100644 --- a/faraday_agent_dispatcher/executor.py +++ b/faraday_agent_dispatcher/executor.py @@ -21,7 +21,7 @@ def __init__(self, name, config): params_section = Sections.EXECUTOR_PARAMS.format(name) varenvs_section = Sections.EXECUTOR_VARENVS.format(name) self.cmd = config.get(executor_section, "cmd") - self.max_size = config[executor_section].get("max_size", 64 * 1024) + self.max_size = int(config[executor_section].get("max_size", 64 * 1024)) self.params = dict(config[params_section]) if params_section in config else {} self.params = {key: value.lower() in ["t", "true"] for key, value in self.params.items()} self.varenvs = dict(config[varenvs_section]) if varenvs_section in config else {} diff --git a/tests/unittests/test_agent_dispatcher.py b/tests/unittests/test_agent_dispatcher.py index f8342fd6..4944043f 100644 --- a/tests/unittests/test_agent_dispatcher.py +++ b/tests/unittests/test_agent_dispatcher.py @@ -128,7 +128,7 @@ async def test_start_and_register(test_config: FaradayTestConfig, tmp_default_co configuration.set(Sections.SERVER, "host", test_config.client.host) configuration.set(Sections.SERVER, "workspace", test_config.workspace) configuration.set(Sections.TOKENS, "registration", test_config.registration_token) - configuration.set(Sections.EXECUTOR_DATA, "cmd", 'exit 1') + configuration.set(Sections.EXECUTOR_DATA.format("ex1"), "cmd", 'exit 1') tmp_default_config.save() # Init and register it @@ -149,7 +149,7 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default configuration.set(Sections.SERVER, "host", test_config.client.host) configuration.set(Sections.SERVER, "workspace", test_config.workspace) configuration.set(Sections.TOKENS, "registration", "NotOk" * 5) - configuration.set(Sections.EXECUTOR, "cmd", 'exit 1') + configuration.set(Sections.EXECUTOR_DATA.format("ex1"), "cmd", 'exit 1') tmp_default_config.save() # Init and register it @@ -179,25 +179,6 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default {"CUT_RESPONSE": "Error: Unrecognized action"} ] }, - { # 2 - "data": {"action": "RUN", "agent_id": 1, "args": {"out": "json"}}, - "logs": [ - {"levelname": "INFO", "msg": "Running executor"}, - {"levelname": "INFO", "msg": "Data sent to bulk create"}, - {"levelname": "INFO", "msg": "Executor finished successfully"} - ], - "ws_responses": [ - { - "action": "RUN_STATUS", - "running": True, - "message": "Running executor from unnamed_agent agent" - }, { - "action": "RUN_STATUS", - "successful": True, - "message": "Executor finished successfully" - } - ] - }, { # 2 "data": {"action": "RUN", "agent_id": 1, "executor": "ex1", "args": {"out": "json"}}, "logs": [ @@ -555,7 +536,7 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default { # 18 "data": { "action": "RUN", "agent_id": 1, - "args": {"out": "json", "WTF": "T"} + "args": {"out": "json"} }, "logs": [ {"levelname": "INFO", "msg": "Running executor", "max_count": 0, @@ -575,7 +556,7 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default { # 19 "data": { "action": "RUN", "agent_id": 1, "executor": "NOT_4N_CORRECT_EXECUTOR", - "args": {"out": "json", "WTF": "T"} + "args": {"out": "json"} }, "logs": [ {"levelname": "INFO", "msg": "Running executor", "max_count": 0, @@ -606,16 +587,19 @@ async def test_run_once(test_config: FaradayTestConfig, tmp_default_config, test Path(__file__).parent.parent / 'data' / 'basic_executor.py' ) - configuration.set(Sections.EXECUTOR, "cmd", "python {}".format(path_to_basic_executor)) - configuration.set(Sections.PARAMS, "out", "True") - [configuration.set(Sections.PARAMS, param, "False") for param in [ + executor_section = Sections.EXECUTOR_DATA.format("ex1") + params_section = Sections.EXECUTOR_PARAMS.format("ex1") + varenvs_section = Sections.EXECUTOR_VARENVS.format("ex1") + configuration.set(executor_section, "cmd", "python {}".format(path_to_basic_executor)) + configuration.set(params_section, "out", "True") + [configuration.set(params_section, param, "False") for param in [ "count", "spare", "spaced_before", "spaced_middle", "err", "fails"]] if "varenvs" in executor_options: for varenv in executor_options["varenvs"]: - configuration.set(Sections.VARENVS, varenv, executor_options["varenvs"][varenv]) + configuration.set(varenvs_section, varenv, executor_options["varenvs"][varenv]) max_size = str(64 * 1024) if "max_size" not in executor_options else executor_options["max_size"] - configuration.set(Sections.EXECUTOR, "max_size", max_size) + configuration.set(executor_section, "max_size", max_size) tmp_default_config.save() From 3bc179fc79751582fdaed1e199f7da3afc9741b2 Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Fri, 8 Nov 2019 13:19:52 -0300 Subject: [PATCH 15/41] [FIX] Tuning little up --- faraday_agent_dispatcher/dispatcher.py | 2 ++ tests/unittests/test_agent_dispatcher.py | 9 +++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/faraday_agent_dispatcher/dispatcher.py b/faraday_agent_dispatcher/dispatcher.py index a855748d..33fecb59 100644 --- a/faraday_agent_dispatcher/dispatcher.py +++ b/faraday_agent_dispatcher/dispatcher.py @@ -169,6 +169,7 @@ async def run_once(self, data:str= None, out_func=None): "message": f"No executor selected to {self.agent_name} agent" }) ) + return if data_dict["executor"] not in self.executors: logger.error("The selected executor not exists") @@ -180,6 +181,7 @@ async def run_once(self, data:str= None, out_func=None): f"agent" }) ) + return executor = self.executors[data_dict["executor"]] diff --git a/tests/unittests/test_agent_dispatcher.py b/tests/unittests/test_agent_dispatcher.py index 4944043f..c4dc8d04 100644 --- a/tests/unittests/test_agent_dispatcher.py +++ b/tests/unittests/test_agent_dispatcher.py @@ -545,12 +545,12 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "min_count": 0}, {"levelname": "INFO", "msg": "Executor finished successfully", "max_count": 0, "min_count": 0}, - {"levelname": "ERROR", "msg": "Unexpected argument passed"}, + {"levelname": "ERROR", "msg": "No executor selected"}, ], "ws_responses": [ {"action": "RUN_STATUS", "running": False, - "message": "No executor selected"} + "message": "No executor selected to unnamed_agent agent"} ] }, { # 19 @@ -565,12 +565,13 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "min_count": 0}, {"levelname": "INFO", "msg": "Executor finished successfully", "max_count": 0, "min_count": 0}, - {"levelname": "ERROR", "msg": "Unexpected argument passed"}, + {"levelname": "ERROR", "msg": "The selected executor not exists"}, ], "ws_responses": [ {"action": "RUN_STATUS", "running": False, - "message": "The selected executor not exists"} + "message": "The selected executor NOT_4N_CORRECT_EXECUTOR not exists in " + "unnamed_agent agent"} ] }, ]) From 3a9450cd5570546ae719b8eaa62960910354ea45 Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Fri, 8 Nov 2019 13:32:27 -0300 Subject: [PATCH 16/41] [ADD] second executor test --- faraday_agent_dispatcher/dispatcher.py | 2 +- tests/unittests/test_agent_dispatcher.py | 50 ++++++++++++++++++------ 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/faraday_agent_dispatcher/dispatcher.py b/faraday_agent_dispatcher/dispatcher.py index 33fecb59..e35ec841 100644 --- a/faraday_agent_dispatcher/dispatcher.py +++ b/faraday_agent_dispatcher/dispatcher.py @@ -181,7 +181,7 @@ async def run_once(self, data:str= None, out_func=None): f"agent" }) ) - return + return executor = self.executors[data_dict["executor"]] diff --git a/tests/unittests/test_agent_dispatcher.py b/tests/unittests/test_agent_dispatcher.py index c4dc8d04..970ddf43 100644 --- a/tests/unittests/test_agent_dispatcher.py +++ b/tests/unittests/test_agent_dispatcher.py @@ -574,6 +574,26 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "unnamed_agent agent"} ] }, + { # 20 + "data": {"action": "RUN", "agent_id": 1, "executor": "ex2", "args": {"out": "json"}}, + "logs": [ + {"levelname": "INFO", "msg": "Running executor"}, + {"levelname": "INFO", "msg": "Data sent to bulk create"}, + {"levelname": "INFO", "msg": "Executor finished successfully"} + ], + "ws_responses": [ + { + "action": "RUN_STATUS", + "running": True, + "message": "Running executor from unnamed_agent agent" + }, { + "action": "RUN_STATUS", + "successful": True, + "message": "Executor finished successfully" + } + ], + "extra": ["ex2"] + }, ]) async def test_run_once(test_config: FaradayTestConfig, tmp_default_config, test_logger_handler, test_logger_folder, executor_options): @@ -588,19 +608,26 @@ async def test_run_once(test_config: FaradayTestConfig, tmp_default_config, test Path(__file__).parent.parent / 'data' / 'basic_executor.py' ) - executor_section = Sections.EXECUTOR_DATA.format("ex1") - params_section = Sections.EXECUTOR_PARAMS.format("ex1") - varenvs_section = Sections.EXECUTOR_VARENVS.format("ex1") - configuration.set(executor_section, "cmd", "python {}".format(path_to_basic_executor)) - configuration.set(params_section, "out", "True") - [configuration.set(params_section, param, "False") for param in [ + executor_names = ["ex1"] + ([] if "extra" not in executor_options else executor_options["extra"]) + configuration.set(Sections.AGENT, "executors", ",".join(executor_names)) + for executor_name in executor_names: + executor_section = Sections.EXECUTOR_DATA.format(executor_name) + params_section = Sections.EXECUTOR_PARAMS.format(executor_name) + varenvs_section = Sections.EXECUTOR_VARENVS.format(executor_name) + for section in [executor_section, params_section, varenvs_section]: + if section not in configuration: + configuration.add_section(section) + + configuration.set(executor_section, "cmd", "python {}".format(path_to_basic_executor)) + configuration.set(params_section, "out", "True") + [configuration.set(params_section, param, "False") for param in [ "count", "spare", "spaced_before", "spaced_middle", "err", "fails"]] - if "varenvs" in executor_options: - for varenv in executor_options["varenvs"]: - configuration.set(varenvs_section, varenv, executor_options["varenvs"][varenv]) + if "varenvs" in executor_options: + for varenv in executor_options["varenvs"]: + configuration.set(varenvs_section, varenv, executor_options["varenvs"][varenv]) - max_size = str(64 * 1024) if "max_size" not in executor_options else executor_options["max_size"] - configuration.set(executor_section, "max_size", max_size) + max_size = str(64 * 1024) if "max_size" not in executor_options else executor_options["max_size"] + configuration.set(executor_section, "max_size", max_size) tmp_default_config.save() @@ -622,6 +649,7 @@ async def ws_messages_checker(msg): min_count, l["msg"] +# TODO CHECK BUILT REPEATED NAME async def test_connect(test_config: FaradayTestConfig, tmp_default_config, test_logger_handler, test_logger_folder): configuration.set(Sections.SERVER, "api_port", str(test_config.client.port)) From 49e933020e195860b28f77907279ae6d6fc2ada2 Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Fri, 8 Nov 2019 14:44:43 -0300 Subject: [PATCH 17/41] [ADD] More verbosing data --- faraday_agent_dispatcher/dispatcher.py | 22 ++-- tests/unittests/test_agent_dispatcher.py | 140 ++++++++++++----------- 2 files changed, 83 insertions(+), 79 deletions(-) diff --git a/faraday_agent_dispatcher/dispatcher.py b/faraday_agent_dispatcher/dispatcher.py index e35ec841..98b50813 100644 --- a/faraday_agent_dispatcher/dispatcher.py +++ b/faraday_agent_dispatcher/dispatcher.py @@ -198,12 +198,13 @@ async def run_once(self, data:str= None, out_func=None): for passed_param in passed_params # For all passed params ]) if not all_accepted: - logger.error("Unexpected argument passed") + logger.error("Unexpected argument passed to {} executor".format(executor.name)) await out_func( json.dumps({ "action": "RUN_STATUS", "running": False, - "message": f"Unexpected argument(s) passed to {self.agent_name} agent" + "message": f"Unexpected argument(s) passed to {executor.name} executor from {self.agent_name} " + f"agent" }) ) mandatory_full = all( @@ -216,18 +217,19 @@ async def run_once(self, data:str= None, out_func=None): ] ) if not mandatory_full: - logger.error("Mandatory argument not passed") + logger.error("Mandatory argument not passed to {} executor".format(executor.name)) await out_func( json.dumps({ "action": "RUN_STATUS", "running": False, - "message": f"Mandatory argument(s) not passed to {self.agent_name} agent" + "message": f"Mandatory argument(s) not passed to {executor.name} executor from " + f"{self.agent_name} agent" }) ) if mandatory_full and all_accepted: - running_msg = f"Running executor from {self.agent_name} agent" - logger.info('Running executor') + running_msg = f"Running {executor.name} executor from {self.agent_name} agent" + logger.info("Running {} executor".format(executor.name)) process = await self.create_process(executor, passed_params) tasks = [StdOutLineProcessor(process, self.session).process_f(), @@ -244,21 +246,21 @@ async def run_once(self, data:str= None, out_func=None): await process.communicate() assert process.returncode is not None if process.returncode == 0: - logger.info("Executor finished successfully") + logger.info("Executor {} finished successfully".format(executor.name)) await out_func( json.dumps({ "action": "RUN_STATUS", "successful": True, - "message": "Executor finished successfully" + "message": f"Executor {executor.name} from {self.agent_name} finished successfully" })) else: logger.warning( - f"Executor finished with exit code {process.returncode}") + f"Executor {executor.name} finished with exit code {process.returncode}") await out_func( json.dumps({ "action": "RUN_STATUS", "successful": False, - "message": "Executor failed" + "message": f"Executor {executor.name} from {self.agent_name} failed" })) async def create_process(self, executor: Executor, args): diff --git a/tests/unittests/test_agent_dispatcher.py b/tests/unittests/test_agent_dispatcher.py index 970ddf43..44a9582f 100644 --- a/tests/unittests/test_agent_dispatcher.py +++ b/tests/unittests/test_agent_dispatcher.py @@ -182,19 +182,19 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default { # 2 "data": {"action": "RUN", "agent_id": 1, "executor": "ex1", "args": {"out": "json"}}, "logs": [ - {"levelname": "INFO", "msg": "Running executor"}, + {"levelname": "INFO", "msg": "Running ex1 executor"}, {"levelname": "INFO", "msg": "Data sent to bulk create"}, - {"levelname": "INFO", "msg": "Executor finished successfully"} + {"levelname": "INFO", "msg": "Executor ex1 finished successfully"} ], "ws_responses": [ { "action": "RUN_STATUS", "running": True, - "message": "Running executor from unnamed_agent agent" + "message": "Running ex1 executor from unnamed_agent agent" }, { "action": "RUN_STATUS", "successful": True, - "message": "Executor finished successfully" + "message": "Executor ex1 from unnamed_agent finished successfully" } ] }, @@ -204,19 +204,19 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "args": {"out": "json", "count": "5"} }, "logs": [ - {"levelname": "INFO", "msg": "Running executor"}, + {"levelname": "INFO", "msg": "Running ex1 executor"}, {"levelname": "ERROR", "msg": "JSON Parsing error: Extra data"}, - {"levelname": "INFO", "msg": "Executor finished successfully"} + {"levelname": "INFO", "msg": "Executor ex1 finished successfully"} ], "ws_responses": [ { "action": "RUN_STATUS", "running": True, - "message": "Running executor from unnamed_agent agent" + "message": "Running ex1 executor from unnamed_agent agent" }, { "action": "RUN_STATUS", "successful": True, - "message": "Executor finished successfully" + "message": "Executor ex1 from unnamed_agent finished successfully" } ] }, @@ -226,19 +226,19 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "args": {"out": "json", "count": "5", "spare": "T"} }, "logs": [ - {"levelname": "INFO", "msg": "Running executor"}, + {"levelname": "INFO", "msg": "Running ex1 executor"}, {"levelname": "INFO", "msg": "Data sent to bulk create", "min_count": 5}, - {"levelname": "INFO", "msg": "Executor finished successfully"} + {"levelname": "INFO", "msg": "Executor ex1 finished successfully"} ], "ws_responses": [ { "action": "RUN_STATUS", "running": True, - "message": "Running executor from unnamed_agent agent" + "message": "Running ex1 executor from unnamed_agent agent" }, { "action": "RUN_STATUS", "successful": True, - "message": "Executor finished successfully" + "message": "Executor ex1 from unnamed_agent finished successfully" } ] }, @@ -250,18 +250,18 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "args": {"out": "json", "spaced_before": "T"} }, "logs": [ - {"levelname": "INFO", "msg": "Running executor"}, - {"levelname": "INFO", "msg": "Executor finished successfully"} + {"levelname": "INFO", "msg": "Running ex1 executor"}, + {"levelname": "INFO", "msg": "Executor ex1 finished successfully"} ], "ws_responses": [ { "action": "RUN_STATUS", "running": True, - "message": "Running executor from unnamed_agent agent" + "message": "Running ex1 executor from unnamed_agent agent" }, { "action": "RUN_STATUS", "successful": True, - "message": "Executor finished successfully" + "message": "Executor ex1 from unnamed_agent finished successfully" } ] }, @@ -271,19 +271,19 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "args": {"out": "json", "spaced_middle": "T", "count": "5", "spare": "T"} }, "logs": [ - {"levelname": "INFO", "msg": "Running executor"}, + {"levelname": "INFO", "msg": "Running ex1 executor"}, {"levelname": "INFO", "msg": "Data sent to bulk create", "max_count": 1}, - {"levelname": "INFO", "msg": "Executor finished successfully"} + {"levelname": "INFO", "msg": "Executor ex1 finished successfully"} ], "ws_responses": [ { "action": "RUN_STATUS", "running": True, - "message": "Running executor from unnamed_agent agent" + "message": "Running ex1 executor from unnamed_agent agent" }, { "action": "RUN_STATUS", "successful": True, - "message": "Executor finished successfully" + "message": "Executor ex1 from unnamed_agent finished successfully" } ] }, @@ -292,21 +292,21 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "action": "RUN", "agent_id": 1, "executor": "ex1", "args": {"out": "bad_json"} }, "logs": [ - {"levelname": "INFO", "msg": "Running executor"}, + {"levelname": "INFO", "msg": "Running ex1 executor"}, {"levelname": "ERROR", "msg": "Invalid data supplied by the executor to the bulk create endpoint. " "Server responded: "}, - {"levelname": "INFO", "msg": "Executor finished successfully"} + {"levelname": "INFO", "msg": "Executor ex1 finished successfully"} ], "ws_responses": [ { "action": "RUN_STATUS", "running": True, - "message": "Running executor from unnamed_agent agent" + "message": "Running ex1 executor from unnamed_agent agent" }, { "action": "RUN_STATUS", "successful": True, - "message": "Executor finished successfully" + "message": "Executor ex1 from unnamed_agent finished successfully" } ] }, @@ -315,19 +315,19 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "action": "RUN", "agent_id": 1, "executor": "ex1", "args": {"out": "str"} }, "logs": [ - {"levelname": "INFO", "msg": "Running executor"}, + {"levelname": "INFO", "msg": "Running ex1 executor"}, {"levelname": "ERROR", "msg": "JSON Parsing error: Expecting value"}, - {"levelname": "INFO", "msg": "Executor finished successfully"} + {"levelname": "INFO", "msg": "Executor ex1 finished successfully"} ], "ws_responses": [ { "action": "RUN_STATUS", "running": True, - "message": "Running executor from unnamed_agent agent" + "message": "Running ex1 executor from unnamed_agent agent" }, { "action": "RUN_STATUS", "successful": True, - "message": "Executor finished successfully" + "message": "Executor ex1 from unnamed_agent finished successfully" } ] }, @@ -337,19 +337,19 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "args": {"out": "none", "err": "T"} }, "logs": [ - {"levelname": "INFO", "msg": "Running executor"}, + {"levelname": "INFO", "msg": "Running ex1 executor"}, {"levelname": "DEBUG", "msg": "Print by stderr"}, - {"levelname": "INFO", "msg": "Executor finished successfully"} + {"levelname": "INFO", "msg": "Executor ex1 finished successfully"} ], "ws_responses": [ { "action": "RUN_STATUS", "running": True, - "message": "Running executor from unnamed_agent agent" + "message": "Running ex1 executor from unnamed_agent agent" }, { "action": "RUN_STATUS", "successful": True, - "message": "Executor finished successfully" + "message": "Executor ex1 from unnamed_agent finished successfully" } ] }, @@ -359,18 +359,18 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "args": {"out": "none", "fails": "T"} }, "logs": [ - {"levelname": "INFO", "msg": "Running executor"}, - {"levelname": "WARNING", "msg": "Executor finished with exit code 1"}, + {"levelname": "INFO", "msg": "Running ex1 executor"}, + {"levelname": "WARNING", "msg": "Executor ex1 finished with exit code 1"}, ], "ws_responses": [ { "action": "RUN_STATUS", "running": True, - "message": "Running executor from unnamed_agent agent" + "message": "Running ex1 executor from unnamed_agent agent" }, { "action": "RUN_STATUS", "successful": False, - "message": "Executor failed" + "message": "Executor ex1 from unnamed_agent failed" } ] }, @@ -382,19 +382,19 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "args": {"out": "none", "err": "T", "fails": "T"} }, "logs": [ - {"levelname": "INFO", "msg": "Running executor"}, + {"levelname": "INFO", "msg": "Running ex1 executor"}, {"levelname": "DEBUG", "msg": "Print by stderr"}, - {"levelname": "WARNING", "msg": "Executor finished with exit code 1"}, + {"levelname": "WARNING", "msg": "Executor ex1 finished with exit code 1"}, ], "ws_responses": [ { "action": "RUN_STATUS", "running": True, - "message": "Running executor from unnamed_agent agent" + "message": "Running ex1 executor from unnamed_agent agent" }, { "action": "RUN_STATUS", "successful": False, - "message": "Executor failed" + "message": "Executor ex1 from unnamed_agent failed" } ] }, @@ -406,21 +406,21 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "args": {"out": "json"} }, "logs": [ - {"levelname": "INFO", "msg": "Running executor"}, + {"levelname": "INFO", "msg": "Running ex1 executor"}, {"levelname": "INFO", "msg": "Data sent to bulk create", "max_count": 0, "min_count": 0}, - {"levelname": "INFO", "msg": "Executor finished successfully"} + {"levelname": "INFO", "msg": "Executor ex1 finished successfully"} ], "varenvs": {"DO_NOTHING": "True"}, "ws_responses": [ { "action": "RUN_STATUS", "running": True, - "message": "Running executor from unnamed_agent agent" + "message": "Running ex1 executor from unnamed_agent agent" }, { "action": "RUN_STATUS", "successful": True, - "message": "Executor finished successfully" + "message": "Executor ex1 from unnamed_agent finished successfully" } ] }, @@ -432,7 +432,7 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "args": {"err": "T", "fails": "T"}, }, "logs": [ - {"levelname": "INFO", "msg": "Running executor", "max_count": 0, + {"levelname": "INFO", "msg": "Running ex1 executor", "max_count": 0, "min_count": 0}, {"levelname": "ERROR", "msg": "Mandatory argument not passed"}, ], @@ -440,7 +440,8 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default { "action": "RUN_STATUS", "running": False, - "message": "Mandatory argument(s) not passed to unnamed_agent agent" + "message": "Mandatory argument(s) not passed to ex1 executor from " + "unnamed_agent agent" } ] }, @@ -450,18 +451,19 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "args": {"out": "json", "WTF": "T"} }, "logs": [ - {"levelname": "INFO", "msg": "Running executor", "max_count": 0, + {"levelname": "INFO", "msg": "Running ex1 executor", "max_count": 0, "min_count": 0}, {"levelname": "INFO", "msg": "Data sent to bulk create", "max_count": 0, "min_count": 0}, - {"levelname": "INFO", "msg": "Executor finished successfully", "max_count": 0, + {"levelname": "INFO", "msg": "Executor ex1 finished successfully", "max_count": 0, "min_count": 0}, {"levelname": "ERROR", "msg": "Unexpected argument passed"}, ], "ws_responses": [ {"action": "RUN_STATUS", "running": False, - "message": "Unexpected argument(s) passed to unnamed_agent agent"} + "message": "Unexpected argument(s) passed to ex1 executor from unnamed_agent " + "agent"} ] }, { # 15 @@ -469,22 +471,22 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "action": "RUN", "agent_id": 1, "executor": "ex1", "args": {"out": "json"} }, "logs": [ - {"levelname": "INFO", "msg": "Running executor"}, + {"levelname": "INFO", "msg": "Running ex1 executor"}, {"levelname": "ERROR", "msg": "Invalid data supplied by the executor to the bulk create endpoint. " "Server responded: "}, - {"levelname": "INFO", "msg": "Executor finished successfully"} + {"levelname": "INFO", "msg": "Executor ex1 finished successfully"} ], "workspace": "error500", "ws_responses": [ { "action": "RUN_STATUS", "running": True, - "message": "Running executor from unnamed_agent agent" + "message": "Running ex1 executor from unnamed_agent agent" }, { "action": "RUN_STATUS", "successful": True, - "message": "Executor finished successfully" + "message": "Executor ex1 from unnamed_agent finished successfully" } ] }, @@ -493,43 +495,43 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "action": "RUN", "agent_id": 1, "executor": "ex1", "args": {"out": "json"} }, "logs": [ - {"levelname": "INFO", "msg": "Running executor"}, + {"levelname": "INFO", "msg": "Running ex1 executor"}, {"levelname": "ERROR", "msg": "Invalid data supplied by the executor to the bulk create endpoint. " "Server responded: "}, - {"levelname": "INFO", "msg": "Executor finished successfully"} + {"levelname": "INFO", "msg": "Executor ex1 finished successfully"} ], "workspace": "error429", "ws_responses": [ { "action": "RUN_STATUS", "running": True, - "message": "Running executor from unnamed_agent agent" + "message": "Running ex1 executor from unnamed_agent agent" }, { "action": "RUN_STATUS", "successful": True, - "message": "Executor finished successfully" + "message": "Executor ex1 from unnamed_agent finished successfully" } ] }, { # 17 "data": {"action": "RUN", "agent_id": 1, "executor": "ex1", "args": {"out": "json"}}, "logs": [ - {"levelname": "INFO", "msg": "Running executor"}, + {"levelname": "INFO", "msg": "Running ex1 executor"}, {"levelname": "ERROR", "msg": "ValueError raised processing stdout, try with " "bigger limiting size in config"}, - {"levelname": "INFO", "msg": "Executor finished successfully"} + {"levelname": "INFO", "msg": "Executor ex1 finished successfully"} ], "max_size": "1", "ws_responses": [ { "action": "RUN_STATUS", "running": True, - "message": "Running executor from unnamed_agent agent" + "message": "Running ex1 executor from unnamed_agent agent" }, { "action": "RUN_STATUS", "successful": True, - "message": "Executor finished successfully" + "message": "Executor ex1 from unnamed_agent finished successfully" } ] }, @@ -539,11 +541,11 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "args": {"out": "json"} }, "logs": [ - {"levelname": "INFO", "msg": "Running executor", "max_count": 0, + {"levelname": "INFO", "msg": "Running ex1 executor", "max_count": 0, "min_count": 0}, {"levelname": "INFO", "msg": "Data sent to bulk create", "max_count": 0, "min_count": 0}, - {"levelname": "INFO", "msg": "Executor finished successfully", "max_count": 0, + {"levelname": "INFO", "msg": "Executor ex1 finished successfully", "max_count": 0, "min_count": 0}, {"levelname": "ERROR", "msg": "No executor selected"}, ], @@ -559,11 +561,11 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "args": {"out": "json"} }, "logs": [ - {"levelname": "INFO", "msg": "Running executor", "max_count": 0, + {"levelname": "INFO", "msg": "Running ex1 executor", "max_count": 0, "min_count": 0}, {"levelname": "INFO", "msg": "Data sent to bulk create", "max_count": 0, "min_count": 0}, - {"levelname": "INFO", "msg": "Executor finished successfully", "max_count": 0, + {"levelname": "INFO", "msg": "Executor ex1 finished successfully", "max_count": 0, "min_count": 0}, {"levelname": "ERROR", "msg": "The selected executor not exists"}, ], @@ -577,19 +579,19 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default { # 20 "data": {"action": "RUN", "agent_id": 1, "executor": "ex2", "args": {"out": "json"}}, "logs": [ - {"levelname": "INFO", "msg": "Running executor"}, + {"levelname": "INFO", "msg": "Running ex2 executor"}, {"levelname": "INFO", "msg": "Data sent to bulk create"}, - {"levelname": "INFO", "msg": "Executor finished successfully"} + {"levelname": "INFO", "msg": "Executor ex2 finished successfully"} ], "ws_responses": [ { "action": "RUN_STATUS", "running": True, - "message": "Running executor from unnamed_agent agent" + "message": "Running ex2 executor from unnamed_agent agent" }, { "action": "RUN_STATUS", "successful": True, - "message": "Executor finished successfully" + "message": "Executor ex2 from unnamed_agent finished successfully" } ], "extra": ["ex2"] From cb2c3dae88d60063862033d064e76c9df5e60dd5 Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Fri, 8 Nov 2019 14:56:58 -0300 Subject: [PATCH 18/41] [FIX] max_size is not mandatory and can be None [ADD] Correct JOIN_AGENT data test --- faraday_agent_dispatcher/dispatcher.py | 4 ++-- faraday_agent_dispatcher/executor.py | 3 +-- .../utils/control_values_utils.py | 20 +++++++++++-------- tests/data/test_config.ini | 1 - tests/unittests/test_agent_dispatcher.py | 19 +++++++++++++----- 5 files changed, 29 insertions(+), 18 deletions(-) diff --git a/faraday_agent_dispatcher/dispatcher.py b/faraday_agent_dispatcher/dispatcher.py index 98b50813..ab6d775b 100644 --- a/faraday_agent_dispatcher/dispatcher.py +++ b/faraday_agent_dispatcher/dispatcher.py @@ -47,8 +47,8 @@ class Dispatcher: __control_dict = { Sections.SERVER: { "host": control_host, - "api_port": control_int, - "websocket_port": control_int, + "api_port": control_int(), + "websocket_port": control_int(), "workspace": control_str }, Sections.TOKENS: { diff --git a/faraday_agent_dispatcher/executor.py b/faraday_agent_dispatcher/executor.py index 02726b11..309cb783 100644 --- a/faraday_agent_dispatcher/executor.py +++ b/faraday_agent_dispatcher/executor.py @@ -10,7 +10,7 @@ class Executor: __control_dict = { Sections.EXECUTOR_DATA: { "cmd": control_str, - "max_size": control_int + "max_size": control_int(True) } } @@ -26,7 +26,6 @@ def __init__(self, name, config): self.params = {key: value.lower() in ["t", "true"] for key, value in self.params.items()} self.varenvs = dict(config[varenvs_section]) if varenvs_section in config else {} - def control_config(self, name, config): for section in self.__control_dict: for option in self.__control_dict[section]: diff --git a/faraday_agent_dispatcher/utils/control_values_utils.py b/faraday_agent_dispatcher/utils/control_values_utils.py index 419f5934..9012ec52 100644 --- a/faraday_agent_dispatcher/utils/control_values_utils.py +++ b/faraday_agent_dispatcher/utils/control_values_utils.py @@ -1,11 +1,15 @@ -import json -def control_int(field_name,value): - if value is None: - raise ValueError(f"Trying to parse {field_name} with None value and should be an int") - try: - int(value) - except ValueError: - raise ValueError(f"Trying to parse {field_name} with value {value} and should be an int") +def control_int(nullable=False): + def control(field_name, value): + if value is None and nullable: + return + if value is None: + raise ValueError(f"Trying to parse {field_name} with None value and should be an int") + try: + int(value) + except ValueError: + raise ValueError(f"Trying to parse {field_name} with value {value} and should be an int") + + return control def control_str(field_name, value): diff --git a/tests/data/test_config.ini b/tests/data/test_config.ini index adb24e2f..a605fe08 100644 --- a/tests/data/test_config.ini +++ b/tests/data/test_config.ini @@ -16,4 +16,3 @@ registration = 1234567890123456789012345 [ex1] ; Complete the cmd option with the command you want the dispatcher to run cmd = exit 1 -max_size = 65536 \ No newline at end of file diff --git a/tests/unittests/test_agent_dispatcher.py b/tests/unittests/test_agent_dispatcher.py index 44a9582f..e7cfe2a2 100644 --- a/tests/unittests/test_agent_dispatcher.py +++ b/tests/unittests/test_agent_dispatcher.py @@ -663,11 +663,20 @@ async def test_connect(test_config: FaradayTestConfig, tmp_default_config, test_ Path(__file__).parent.parent / 'data' / 'basic_executor.py' ) - configuration.set(Sections.EXECUTOR, "cmd", "python {}".format(path_to_basic_executor)) - configuration.set(Sections.EXECUTOR, "executors", "[ex1,ex2,ex3]") - configuration.set(Sections.PARAMS, "out", "True") - [configuration.set(Sections.PARAMS, param, "False") for param in [ - "count", "spare", "spaced_before", "spaced_middle", "err", "fails"]] + configuration.set(Sections.AGENT, "executors", "ex1,ex2,ex3") + + for executor_name in ["ex1","ex2","ex3"]: + executor_section = Sections.EXECUTOR_DATA.format(executor_name) + params_section = Sections.EXECUTOR_PARAMS.format(executor_name) + for section in [executor_section, params_section]: + if section not in configuration: + configuration.add_section(section) + configuration.set(executor_section, "cmd", "python {}".format(path_to_basic_executor)) + + configuration.set(Sections.EXECUTOR_PARAMS.format("ex1"), "param1", "True") + configuration.set(Sections.EXECUTOR_PARAMS.format("ex1"), "param2", "False") + configuration.set(Sections.EXECUTOR_PARAMS.format("ex2"), "param3", "False") + configuration.set(Sections.EXECUTOR_PARAMS.format("ex2"), "param4", "False") tmp_default_config.save() dispatcher = Dispatcher(test_config.client.session, tmp_default_config.config_file_path) From 9a8411ce29672ee5ed05931626a33409834ed185 Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Fri, 8 Nov 2019 15:04:54 -0300 Subject: [PATCH 19/41] [ADD] JOIN_AGENT now sends executor data --- faraday_agent_dispatcher/dispatcher.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/faraday_agent_dispatcher/dispatcher.py b/faraday_agent_dispatcher/dispatcher.py index ab6d775b..943b3cbe 100644 --- a/faraday_agent_dispatcher/dispatcher.py +++ b/faraday_agent_dispatcher/dispatcher.py @@ -118,13 +118,15 @@ async def register(self): async def connect(self, out_func=None): - if not self.websocket_token: + if not self.websocket_token and not out_func: return connected_data = json.dumps({ 'action': 'JOIN_AGENT', 'workspace': self.workspace, 'token': self.websocket_token, + 'executors': [{"executor_name": executor.name, "args": executor.params} + for executor in self.executors.values()] }) if out_func is None: From e758522ae54cf1deb334d7e64ff9acb3d99da5a4 Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Fri, 8 Nov 2019 16:01:47 -0300 Subject: [PATCH 20/41] [ADD] Test and fix, cannot repeat executors names --- faraday_agent_dispatcher/dispatcher.py | 2 +- .../utils/control_values_utils.py | 13 ++++++++++--- tests/unittests/test_agent_dispatcher.py | 5 ++++- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/faraday_agent_dispatcher/dispatcher.py b/faraday_agent_dispatcher/dispatcher.py index 943b3cbe..64dbd380 100644 --- a/faraday_agent_dispatcher/dispatcher.py +++ b/faraday_agent_dispatcher/dispatcher.py @@ -57,7 +57,7 @@ class Dispatcher: }, Sections.AGENT: { "agent_name": control_str, - "executors": control_list + "executors": control_list(can_repeat=False) }, } diff --git a/faraday_agent_dispatcher/utils/control_values_utils.py b/faraday_agent_dispatcher/utils/control_values_utils.py index 9012ec52..16b01786 100644 --- a/faraday_agent_dispatcher/utils/control_values_utils.py +++ b/faraday_agent_dispatcher/utils/control_values_utils.py @@ -21,9 +21,16 @@ def control_host(field_name, value): control_str(field_name, value) -def control_list(field_name, value): - if not isinstance(value, str) or not isinstance(value.split(","),list): - raise ValueError(f"Trying to parse {field_name} with value {value} and should be a list") + +def control_list(can_repeat=True): + def control(field_name, value): + if not isinstance(value, str): + raise ValueError(f"Trying to parse {field_name} with value {value} and should be a list") + listt = value.split(",") + if len(listt) != len(set(listt)): + raise ValueError(f"Trying to parse {field_name} with value {value} and contains repeated values") + + return control def control_bool(field_name, value): diff --git a/tests/unittests/test_agent_dispatcher.py b/tests/unittests/test_agent_dispatcher.py index e7cfe2a2..5b252938 100644 --- a/tests/unittests/test_agent_dispatcher.py +++ b/tests/unittests/test_agent_dispatcher.py @@ -102,6 +102,10 @@ {"remove": {Sections.AGENT: ["agent_name"]}, "replace": {}, "expected_exception": ValueError}, + {"remove": {}, + "replace": {Sections.AGENT: {"executors": "ex1,ex1"}}, + "expected_exception": ValueError + }, {"remove": {}, "replace": {}} ]) @@ -651,7 +655,6 @@ async def ws_messages_checker(msg): min_count, l["msg"] -# TODO CHECK BUILT REPEATED NAME async def test_connect(test_config: FaradayTestConfig, tmp_default_config, test_logger_handler, test_logger_folder): configuration.set(Sections.SERVER, "api_port", str(test_config.client.port)) From 8b6db05b5428227184c6ecb8fcefa5832659a385 Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Wed, 20 Nov 2019 16:28:01 -0300 Subject: [PATCH 21/41] [ADD] Executor name in RUN_STATUS dict tests --- tests/unittests/test_agent_dispatcher.py | 60 +++++++++++++++++++----- 1 file changed, 49 insertions(+), 11 deletions(-) diff --git a/tests/unittests/test_agent_dispatcher.py b/tests/unittests/test_agent_dispatcher.py index 5b252938..f9c99f05 100644 --- a/tests/unittests/test_agent_dispatcher.py +++ b/tests/unittests/test_agent_dispatcher.py @@ -193,10 +193,12 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "ws_responses": [ { "action": "RUN_STATUS", + "executor_name": "ex1", "running": True, "message": "Running ex1 executor from unnamed_agent agent" }, { "action": "RUN_STATUS", + "executor_name": "ex1", "successful": True, "message": "Executor ex1 from unnamed_agent finished successfully" } @@ -215,10 +217,12 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "ws_responses": [ { "action": "RUN_STATUS", + "executor_name": "ex1", "running": True, "message": "Running ex1 executor from unnamed_agent agent" }, { "action": "RUN_STATUS", + "executor_name": "ex1", "successful": True, "message": "Executor ex1 from unnamed_agent finished successfully" } @@ -237,10 +241,12 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "ws_responses": [ { "action": "RUN_STATUS", + "executor_name": "ex1", "running": True, "message": "Running ex1 executor from unnamed_agent agent" }, { "action": "RUN_STATUS", + "executor_name": "ex1", "successful": True, "message": "Executor ex1 from unnamed_agent finished successfully" } @@ -260,10 +266,12 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "ws_responses": [ { "action": "RUN_STATUS", + "executor_name": "ex1", "running": True, "message": "Running ex1 executor from unnamed_agent agent" }, { "action": "RUN_STATUS", + "executor_name": "ex1", "successful": True, "message": "Executor ex1 from unnamed_agent finished successfully" } @@ -282,10 +290,12 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "ws_responses": [ { "action": "RUN_STATUS", + "executor_name": "ex1", "running": True, "message": "Running ex1 executor from unnamed_agent agent" }, { "action": "RUN_STATUS", + "executor_name": "ex1", "successful": True, "message": "Executor ex1 from unnamed_agent finished successfully" } @@ -305,10 +315,12 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "ws_responses": [ { "action": "RUN_STATUS", + "executor_name": "ex1", "running": True, "message": "Running ex1 executor from unnamed_agent agent" }, { "action": "RUN_STATUS", + "executor_name": "ex1", "successful": True, "message": "Executor ex1 from unnamed_agent finished successfully" } @@ -326,10 +338,12 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "ws_responses": [ { "action": "RUN_STATUS", + "executor_name": "ex1", "running": True, "message": "Running ex1 executor from unnamed_agent agent" }, { "action": "RUN_STATUS", + "executor_name": "ex1", "successful": True, "message": "Executor ex1 from unnamed_agent finished successfully" } @@ -348,10 +362,12 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "ws_responses": [ { "action": "RUN_STATUS", + "executor_name": "ex1", "running": True, "message": "Running ex1 executor from unnamed_agent agent" }, { "action": "RUN_STATUS", + "executor_name": "ex1", "successful": True, "message": "Executor ex1 from unnamed_agent finished successfully" } @@ -369,10 +385,12 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "ws_responses": [ { "action": "RUN_STATUS", + "executor_name": "ex1", "running": True, "message": "Running ex1 executor from unnamed_agent agent" }, { "action": "RUN_STATUS", + "executor_name": "ex1", "successful": False, "message": "Executor ex1 from unnamed_agent failed" } @@ -393,10 +411,12 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "ws_responses": [ { "action": "RUN_STATUS", + "executor_name": "ex1", "running": True, "message": "Running ex1 executor from unnamed_agent agent" }, { "action": "RUN_STATUS", + "executor_name": "ex1", "successful": False, "message": "Executor ex1 from unnamed_agent failed" } @@ -419,10 +439,12 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "ws_responses": [ { "action": "RUN_STATUS", + "executor_name": "ex1", "running": True, "message": "Running ex1 executor from unnamed_agent agent" }, { "action": "RUN_STATUS", + "executor_name": "ex1", "successful": True, "message": "Executor ex1 from unnamed_agent finished successfully" } @@ -443,6 +465,7 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "ws_responses": [ { "action": "RUN_STATUS", + "executor_name": "ex1", "running": False, "message": "Mandatory argument(s) not passed to ex1 executor from " "unnamed_agent agent" @@ -464,10 +487,13 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default {"levelname": "ERROR", "msg": "Unexpected argument passed"}, ], "ws_responses": [ - {"action": "RUN_STATUS", - "running": False, - "message": "Unexpected argument(s) passed to ex1 executor from unnamed_agent " - "agent"} + { + "action": "RUN_STATUS", + "executor_name": "ex1", + "running": False, + "message": "Unexpected argument(s) passed to ex1 executor from unnamed_agent " + "agent" + } ] }, { # 15 @@ -485,10 +511,12 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "ws_responses": [ { "action": "RUN_STATUS", + "executor_name": "ex1", "running": True, "message": "Running ex1 executor from unnamed_agent agent" }, { "action": "RUN_STATUS", + "executor_name": "ex1", "successful": True, "message": "Executor ex1 from unnamed_agent finished successfully" } @@ -509,10 +537,12 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "ws_responses": [ { "action": "RUN_STATUS", + "executor_name": "ex1", "running": True, "message": "Running ex1 executor from unnamed_agent agent" }, { "action": "RUN_STATUS", + "executor_name": "ex1", "successful": True, "message": "Executor ex1 from unnamed_agent finished successfully" } @@ -530,10 +560,12 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "ws_responses": [ { "action": "RUN_STATUS", + "executor_name": "ex1", "running": True, "message": "Running ex1 executor from unnamed_agent agent" }, { "action": "RUN_STATUS", + "executor_name": "ex1", "successful": True, "message": "Executor ex1 from unnamed_agent finished successfully" } @@ -554,9 +586,11 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default {"levelname": "ERROR", "msg": "No executor selected"}, ], "ws_responses": [ - {"action": "RUN_STATUS", - "running": False, - "message": "No executor selected to unnamed_agent agent"} + { + "action": "RUN_STATUS", + "running": False, + "message": "No executor selected to unnamed_agent agent" + } ] }, { # 19 @@ -574,10 +608,12 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default {"levelname": "ERROR", "msg": "The selected executor not exists"}, ], "ws_responses": [ - {"action": "RUN_STATUS", - "running": False, - "message": "The selected executor NOT_4N_CORRECT_EXECUTOR not exists in " - "unnamed_agent agent"} + { + "action": "RUN_STATUS", + "executor_name": "NOT_4N_CORRECT_EXECUTOR", + "running": False, + "message": "The selected executor NOT_4N_CORRECT_EXECUTOR not exists in " + "unnamed_agent agent"} ] }, { # 20 @@ -590,10 +626,12 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default "ws_responses": [ { "action": "RUN_STATUS", + "executor_name": "ex2", "running": True, "message": "Running ex2 executor from unnamed_agent agent" }, { "action": "RUN_STATUS", + "executor_name": "ex2", "successful": True, "message": "Executor ex2 from unnamed_agent finished successfully" } From b3a4f6e07b3903b10150b40414a80bd3d09fe569 Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Wed, 20 Nov 2019 16:29:13 -0300 Subject: [PATCH 22/41] [ADD] Executor names in RUN_STATUS ws response --- faraday_agent_dispatcher/dispatcher.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/faraday_agent_dispatcher/dispatcher.py b/faraday_agent_dispatcher/dispatcher.py index 64dbd380..9bb079e2 100644 --- a/faraday_agent_dispatcher/dispatcher.py +++ b/faraday_agent_dispatcher/dispatcher.py @@ -178,6 +178,7 @@ async def run_once(self, data:str= None, out_func=None): await out_func( json.dumps({ "action": "RUN_STATUS", + "executor_name": data_dict['executor'], "running": False, "message": f"The selected executor {data_dict['executor']} not exists in {self.agent_name} " f"agent" @@ -204,6 +205,7 @@ async def run_once(self, data:str= None, out_func=None): await out_func( json.dumps({ "action": "RUN_STATUS", + "executor_name": executor.name, "running": False, "message": f"Unexpected argument(s) passed to {executor.name} executor from {self.agent_name} " f"agent" @@ -223,6 +225,7 @@ async def run_once(self, data:str= None, out_func=None): await out_func( json.dumps({ "action": "RUN_STATUS", + "executor_name": executor.name, "running": False, "message": f"Mandatory argument(s) not passed to {executor.name} executor from " f"{self.agent_name} agent" @@ -240,6 +243,7 @@ async def run_once(self, data:str= None, out_func=None): await out_func( json.dumps({ "action": "RUN_STATUS", + "executor_name": executor.name, "running": True, "message": running_msg }) @@ -252,6 +256,7 @@ async def run_once(self, data:str= None, out_func=None): await out_func( json.dumps({ "action": "RUN_STATUS", + "executor_name": executor.name, "successful": True, "message": f"Executor {executor.name} from {self.agent_name} finished successfully" })) @@ -261,6 +266,7 @@ async def run_once(self, data:str= None, out_func=None): await out_func( json.dumps({ "action": "RUN_STATUS", + "executor_name": executor.name, "successful": False, "message": f"Executor {executor.name} from {self.agent_name} failed" })) From e03313fe7892a2e358298e17b289e243c0ea0008 Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Wed, 27 Nov 2019 10:30:13 -0300 Subject: [PATCH 23/41] [ADD] Tests for not present section --- faraday_agent_dispatcher/dispatcher.py | 4 ++++ tests/unittests/test_agent_dispatcher.py | 19 +++++++++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/faraday_agent_dispatcher/dispatcher.py b/faraday_agent_dispatcher/dispatcher.py index 9bb079e2..85c4e9e2 100644 --- a/faraday_agent_dispatcher/dispatcher.py +++ b/faraday_agent_dispatcher/dispatcher.py @@ -294,5 +294,9 @@ async def create_process(self, executor: Executor, args): def control_config(self): for section in self.__control_dict: for option in self.__control_dict[section]: + # if section not in config: + # err = f"Section {section} is an mandatory section in the config" # TODO "run config cmd" + # logger.error(err) + # raise ValueError(err) value = config.get(section, option) if option in config[section] else None self.__control_dict[section][option](option, value) diff --git a/tests/unittests/test_agent_dispatcher.py b/tests/unittests/test_agent_dispatcher.py index f9c99f05..70fb20cd 100644 --- a/tests/unittests/test_agent_dispatcher.py +++ b/tests/unittests/test_agent_dispatcher.py @@ -106,6 +106,18 @@ "replace": {Sections.AGENT: {"executors": "ex1,ex1"}}, "expected_exception": ValueError }, + {"remove": {Sections.AGENT: ["section"]}, + "replace": {}, + "expected_exception": ValueError + }, + {"remove": {Sections.TOKENS: ["section"]}, + "replace": {}, + "expected_exception": ValueError + }, + {"remove": {Sections.SERVER: ["section"]}, + "replace": {}, + "expected_exception": ValueError + }, {"remove": {}, "replace": {}} ]) @@ -116,8 +128,11 @@ def test_basic_built(tmp_custom_config, config_changes_dict): configuration.add_section(section) configuration.set(section, option, config_changes_dict["replace"][section][option]) for section in config_changes_dict["remove"]: - for option in config_changes_dict["remove"][section]: - configuration.remove_option(section, option) + if "section" in config_changes_dict["remove"][section]: + configuration.remove_section(section) + else: + for option in config_changes_dict["remove"][section]: + configuration.remove_option(section, option) tmp_custom_config.save() if "expected_exception" in config_changes_dict: with pytest.raises(config_changes_dict["expected_exception"]): From 21f602ac4b5bb13d51e90c18801e30049e16a1bf Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Wed, 27 Nov 2019 10:31:27 -0300 Subject: [PATCH 24/41] [ADD] Fix section not present --- faraday_agent_dispatcher/dispatcher.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/faraday_agent_dispatcher/dispatcher.py b/faraday_agent_dispatcher/dispatcher.py index 85c4e9e2..c3c7f66a 100644 --- a/faraday_agent_dispatcher/dispatcher.py +++ b/faraday_agent_dispatcher/dispatcher.py @@ -294,9 +294,9 @@ async def create_process(self, executor: Executor, args): def control_config(self): for section in self.__control_dict: for option in self.__control_dict[section]: - # if section not in config: - # err = f"Section {section} is an mandatory section in the config" # TODO "run config cmd" - # logger.error(err) - # raise ValueError(err) + if section not in config: + err = f"Section {section} is an mandatory section in the config" # TODO "run config cmd" + logger.error(err) + raise ValueError(err) value = config.get(section, option) if option in config[section] else None self.__control_dict[section][option](option, value) From 8d1444e6dbeee9e6910011cc276d81488b2ab73b Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Wed, 27 Nov 2019 11:04:36 -0300 Subject: [PATCH 25/41] [ADD] Tests for spaced executor names --- tests/unittests/test_agent_dispatcher.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/unittests/test_agent_dispatcher.py b/tests/unittests/test_agent_dispatcher.py index f9c99f05..42ea352b 100644 --- a/tests/unittests/test_agent_dispatcher.py +++ b/tests/unittests/test_agent_dispatcher.py @@ -106,6 +106,19 @@ "replace": {Sections.AGENT: {"executors": "ex1,ex1"}}, "expected_exception": ValueError }, + {"remove": {}, + "replace": {Sections.AGENT: {"executors": "ex1, ex2"}}, + }, + {"remove": {}, + "replace": {Sections.AGENT: {"executors": "ex1,ex2 "}}, + }, + {"remove": {}, + "replace": {Sections.AGENT: {"executors": " ex1,ex2"}}, + }, + {"remove": {}, + "replace": {Sections.AGENT: {"executors": "ex1,ex 1"}}, + "expected_exception": ValueError + }, {"remove": {}, "replace": {}} ]) From 1c38ddb9cb714380592fe81e3a1d7d0a12d17101 Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Wed, 27 Nov 2019 11:07:15 -0300 Subject: [PATCH 26/41] [ADD] More tests --- tests/unittests/test_agent_dispatcher.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/unittests/test_agent_dispatcher.py b/tests/unittests/test_agent_dispatcher.py index 42ea352b..59f5dfa1 100644 --- a/tests/unittests/test_agent_dispatcher.py +++ b/tests/unittests/test_agent_dispatcher.py @@ -115,6 +115,9 @@ {"remove": {}, "replace": {Sections.AGENT: {"executors": " ex1,ex2"}}, }, + {"remove": {}, + "replace": {Sections.AGENT: {"executors": " ex1, ex2 , ex3"}}, + }, {"remove": {}, "replace": {Sections.AGENT: {"executors": "ex1,ex 1"}}, "expected_exception": ValueError From ff929cc4d9843c8750dc7605164ff9f71c8ebba5 Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Wed, 27 Nov 2019 11:37:02 -0300 Subject: [PATCH 27/41] [FIX] Space in executor names --- faraday_agent_dispatcher/executor.py | 6 +++++- tests/data/test_config.ini | 8 ++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/faraday_agent_dispatcher/executor.py b/faraday_agent_dispatcher/executor.py index 309cb783..527c9b2f 100644 --- a/faraday_agent_dispatcher/executor.py +++ b/faraday_agent_dispatcher/executor.py @@ -14,7 +14,8 @@ class Executor: } } - def __init__(self, name, config): + def __init__(self, name: str, config): + name = name.strip() self.control_config(name, config) self.name = name executor_section = Sections.EXECUTOR_DATA.format(name) @@ -27,6 +28,9 @@ def __init__(self, name, config): self.varenvs = dict(config[varenvs_section]) if varenvs_section in config else {} def control_config(self, name, config): + if " " in name: + raise ValueError(f"Executor names can't contains space character, passed name: {name}") + for section in self.__control_dict: for option in self.__control_dict[section]: value = config.get(section.format(name), option) if option in config[section.format(name)] else None diff --git a/tests/data/test_config.ini b/tests/data/test_config.ini index a605fe08..f88c7df0 100644 --- a/tests/data/test_config.ini +++ b/tests/data/test_config.ini @@ -16,3 +16,11 @@ registration = 1234567890123456789012345 [ex1] ; Complete the cmd option with the command you want the dispatcher to run cmd = exit 1 + +[ex2] +; Complete the cmd option with the command you want the dispatcher to run +cmd = exit 1 + +[ex3] +; Complete the cmd option with the command you want the dispatcher to run +cmd = exit 1 From 5fcb5718857fcc4eb743f07f31d69e1eb39dd5ff Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Wed, 27 Nov 2019 11:54:33 -0300 Subject: [PATCH 28/41] [TYPO] --- tests/unittests/test_agent_dispatcher.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/unittests/test_agent_dispatcher.py b/tests/unittests/test_agent_dispatcher.py index 59f5dfa1..0d6b68e7 100644 --- a/tests/unittests/test_agent_dispatcher.py +++ b/tests/unittests/test_agent_dispatcher.py @@ -633,26 +633,26 @@ async def test_start_with_bad_config(test_config: FaradayTestConfig, tmp_default ] }, { # 20 - "data": {"action": "RUN", "agent_id": 1, "executor": "ex2", "args": {"out": "json"}}, + "data": {"action": "RUN", "agent_id": 1, "executor": "add_ex1", "args": {"out": "json"}}, "logs": [ - {"levelname": "INFO", "msg": "Running ex2 executor"}, + {"levelname": "INFO", "msg": "Running add_ex1 executor"}, {"levelname": "INFO", "msg": "Data sent to bulk create"}, - {"levelname": "INFO", "msg": "Executor ex2 finished successfully"} + {"levelname": "INFO", "msg": "Executor add_ex1 finished successfully"} ], "ws_responses": [ { "action": "RUN_STATUS", - "executor_name": "ex2", + "executor_name": "add_ex1", "running": True, - "message": "Running ex2 executor from unnamed_agent agent" + "message": "Running add_ex1 executor from unnamed_agent agent" }, { "action": "RUN_STATUS", - "executor_name": "ex2", + "executor_name": "add_ex1", "successful": True, - "message": "Executor ex2 from unnamed_agent finished successfully" + "message": "Executor add_ex1 from unnamed_agent finished successfully" } ], - "extra": ["ex2"] + "extra": ["add_ex1"] }, ]) async def test_run_once(test_config: FaradayTestConfig, tmp_default_config, test_logger_handler, From 571626f36827f0629f7e790a811306ab9e6fafac Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Wed, 27 Nov 2019 12:05:00 -0300 Subject: [PATCH 29/41] [MOD] basic_executor.py with prefix --- tests/data/basic_executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data/basic_executor.py b/tests/data/basic_executor.py index e6056eb2..57d2271d 100644 --- a/tests/data/basic_executor.py +++ b/tests/data/basic_executor.py @@ -51,7 +51,7 @@ spaced_before = os.getenv("SPACED_BEFORE") is not None spaced_middle = os.getenv("SPACED_MIDDLE") is not None spare = os.getenv("SPARE") is not None - omit_everything = os.getenv("DO_NOTHING", None) + omit_everything = os.getenv("EXECUTOR_CONFIG_DO_NOTHING", None) if out and omit_everything is None: host_data_ = host_data.copy() host_data_['vulnerabilities'] = [vuln_data] From 23bb4c315db44973c7b2637a602f329369ed65f4 Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Wed, 27 Nov 2019 12:07:39 -0300 Subject: [PATCH 30/41] [ADD] Varenv prefix --- faraday_agent_dispatcher/dispatcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/faraday_agent_dispatcher/dispatcher.py b/faraday_agent_dispatcher/dispatcher.py index 9bb079e2..8e29b07f 100644 --- a/faraday_agent_dispatcher/dispatcher.py +++ b/faraday_agent_dispatcher/dispatcher.py @@ -280,7 +280,7 @@ async def create_process(self, executor: Executor, args): logger.error("Args from data received has a not supported type") raise ValueError("Args from data received has a not supported type") for varenv, value in executor.varenvs.items(): - env[varenv.upper()] = value + env[f"EXECUTOR_CONFIG_{varenv.upper()}"] = value process = await asyncio.create_subprocess_shell( executor.cmd, stdout=asyncio.subprocess.PIPE, From 8a48a63f9a5d35551d6d6eada010aeea6b70379a Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Wed, 27 Nov 2019 12:10:27 -0300 Subject: [PATCH 31/41] [ADD] Test executor name in list but not section --- tests/unittests/test_agent_dispatcher.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/unittests/test_agent_dispatcher.py b/tests/unittests/test_agent_dispatcher.py index 0d6b68e7..c10f3920 100644 --- a/tests/unittests/test_agent_dispatcher.py +++ b/tests/unittests/test_agent_dispatcher.py @@ -122,6 +122,10 @@ "replace": {Sections.AGENT: {"executors": "ex1,ex 1"}}, "expected_exception": ValueError }, + {"remove": {}, + "replace": {Sections.AGENT: {"executors": "ex1,ex8"}}, + "expected_exception": ValueError + }, {"remove": {}, "replace": {}} ]) From 853aef81006f61a22d3707612b8d85673ba8b18e Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 27 Nov 2019 12:35:28 -0300 Subject: [PATCH 32/41] update nmap script to use params --- contrib/nmap.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/contrib/nmap.py b/contrib/nmap.py index a86d8ae8..481eb7b2 100755 --- a/contrib/nmap.py +++ b/contrib/nmap.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +import os import subprocess """Yo need to clone and install faraday plugins""" @@ -6,12 +7,13 @@ cmd = [ "nmap", - "-p80,443", - "190.210.92.77", + "-p{}".format(os.environ.get('PORT_LIST')), + os.environ.get('TARGET'), "-oX", "-", ] + results = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) nmap = NmapPlugin() nmap.parseOutputString(results.stdout) From f1752dda13aff85fbb8fabfacb0d524672a9bc4c Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Wed, 27 Nov 2019 13:47:55 -0300 Subject: [PATCH 33/41] [FIX] If name in the list, section must exists --- faraday_agent_dispatcher/executor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/faraday_agent_dispatcher/executor.py b/faraday_agent_dispatcher/executor.py index 527c9b2f..77386ea2 100644 --- a/faraday_agent_dispatcher/executor.py +++ b/faraday_agent_dispatcher/executor.py @@ -30,6 +30,8 @@ def __init__(self, name: str, config): def control_config(self, name, config): if " " in name: raise ValueError(f"Executor names can't contains space character, passed name: {name}") + if Sections.EXECUTOR_DATA.format(name) not in config: + raise ValueError(f"{name} is an executor name but there is no proper section") for section in self.__control_dict: for option in self.__control_dict[section]: From 4696932e90cfaec626217773dfe0f0305fb25777 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 27 Nov 2019 14:58:15 -0300 Subject: [PATCH 34/41] use prefix on nmap script --- contrib/nmap.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/nmap.py b/contrib/nmap.py index 481eb7b2..3420a164 100755 --- a/contrib/nmap.py +++ b/contrib/nmap.py @@ -7,8 +7,8 @@ cmd = [ "nmap", - "-p{}".format(os.environ.get('PORT_LIST')), - os.environ.get('TARGET'), + "-p{}".format(os.environ.get('EXECUTOR_CONFIG_PORT_LIST')), + os.environ.get('EXECUTOR_CONFIG_TARGET'), "-oX", "-", ] From cc02b334de805ce64c2c60341f7a5e2165ba8bf7 Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Wed, 27 Nov 2019 15:09:10 -0300 Subject: [PATCH 35/41] [ADD] Test of duplicated section --- tests/unittests/test_agent_dispatcher.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/unittests/test_agent_dispatcher.py b/tests/unittests/test_agent_dispatcher.py index f9c99f05..3f271560 100644 --- a/tests/unittests/test_agent_dispatcher.py +++ b/tests/unittests/test_agent_dispatcher.py @@ -25,6 +25,7 @@ from pathlib import Path from itsdangerous import TimestampSigner +from configparser import DuplicateSectionError from faraday_agent_dispatcher.dispatcher import Dispatcher from faraday_agent_dispatcher.config import ( @@ -106,6 +107,11 @@ "replace": {Sections.AGENT: {"executors": "ex1,ex1"}}, "expected_exception": ValueError }, + {"remove": {}, + "replace": {}, + "duplicate_exception": True, + "expected_exception": ValueError + }, {"remove": {}, "replace": {}} ]) @@ -120,6 +126,12 @@ def test_basic_built(tmp_custom_config, config_changes_dict): configuration.remove_option(section, option) tmp_custom_config.save() if "expected_exception" in config_changes_dict: + if "duplicate_exception" in config_changes_dict and config_changes_dict["duplicate_exception"]: + with open(tmp_custom_config.config_file_path, "r") as file: + content = file.read() + with open(tmp_custom_config.config_file_path, "w") as file: + file.write(content) + file.write(content) with pytest.raises(config_changes_dict["expected_exception"]): Dispatcher(None, tmp_custom_config.config_file_path) else: From b0d138e6aca046d9e8f96f833c8b8fe1a6edd067 Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Wed, 27 Nov 2019 15:12:11 -0300 Subject: [PATCH 36/41] [FIX] Control duplicated sections --- faraday_agent_dispatcher/config.py | 8 ++++++-- tests/unittests/test_agent_dispatcher.py | 1 - 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/faraday_agent_dispatcher/config.py b/faraday_agent_dispatcher/config.py index ac3e5a5f..de846897 100644 --- a/faraday_agent_dispatcher/config.py +++ b/faraday_agent_dispatcher/config.py @@ -17,6 +17,7 @@ import logging import configparser from pathlib import Path +from configparser import DuplicateSectionError try: FARADAY_PATH = Path(os.environ['FARADAY_HOME']) @@ -39,8 +40,11 @@ def reset_config(filepath): instance.clear() - if not instance.read(filepath): - raise ValueError(f'Unable to read config file located at {filepath}') + try: + if not instance.read(filepath): + raise ValueError(f'Unable to read config file located at {filepath}') + except DuplicateSectionError as e: + raise ValueError(f'The config in {filepath} contains duplicated sections') def check_filepath(filepath: str = None): diff --git a/tests/unittests/test_agent_dispatcher.py b/tests/unittests/test_agent_dispatcher.py index 3f271560..bfdbb2ce 100644 --- a/tests/unittests/test_agent_dispatcher.py +++ b/tests/unittests/test_agent_dispatcher.py @@ -25,7 +25,6 @@ from pathlib import Path from itsdangerous import TimestampSigner -from configparser import DuplicateSectionError from faraday_agent_dispatcher.dispatcher import Dispatcher from faraday_agent_dispatcher.config import ( From b78b4df5796ff95268432fca1fdb0fcc8134001b Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Thu, 28 Nov 2019 10:12:01 -0300 Subject: [PATCH 37/41] [MOD] Tests of PARAMS prefix --- tests/data/basic_executor.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/tests/data/basic_executor.py b/tests/data/basic_executor.py index 57d2271d..a3fa208c 100644 --- a/tests/data/basic_executor.py +++ b/tests/data/basic_executor.py @@ -36,22 +36,15 @@ 'refs': ['CVE-1234'] } -import argparse # TODO REMOVE WHEN FARADAY SENDS ARGS - if __name__ == '__main__': - ##### TODO REMOVE WHEN FARADAY SENDS ARGS - parser = argparse.ArgumentParser() - parser.add_argument('--out', action='store', help='if set prints by stdout') - args = parser.parse_args() - ##### TODO REMOVE WHEN FARADAY SENDS ARGS - out = os.getenv("OUT", args.out or None) - count = os.getenv("COUNT", 1) - err = os.getenv("ERR") is not None - fails = os.getenv("FAILS") is not None - spaced_before = os.getenv("SPACED_BEFORE") is not None - spaced_middle = os.getenv("SPACED_MIDDLE") is not None - spare = os.getenv("SPARE") is not None - omit_everything = os.getenv("EXECUTOR_CONFIG_DO_NOTHING", None) + out = os.getenv("EXECUTOR_CONFIG_OUT") + count = os.getenv("EXECUTOR_CONFIG_COUNT", 1) + err = os.getenv("EXECUTOR_CONFIG_ERR") is not None + fails = os.getenv("EXECUTOR_CONFIG_FAILS") is not None + spaced_before = os.getenv("EXECUTOR_CONFIG_SPACED_BEFORE") is not None + spaced_middle = os.getenv("EXECUTOR_CONFIG_SPACED_MIDDLE") is not None + spare = os.getenv("EXECUTOR_CONFIG_SPARE") is not None + omit_everything = os.getenv("DO_NOTHING", None) if out and omit_everything is None: host_data_ = host_data.copy() host_data_['vulnerabilities'] = [vuln_data] From d942c9c3dd9648ca50bc6a97e47076752a26f4db Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Thu, 28 Nov 2019 10:13:43 -0300 Subject: [PATCH 38/41] [FIX] PARAMS prefix --- faraday_agent_dispatcher/dispatcher.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/faraday_agent_dispatcher/dispatcher.py b/faraday_agent_dispatcher/dispatcher.py index 0fe9cd2f..027bc19c 100644 --- a/faraday_agent_dispatcher/dispatcher.py +++ b/faraday_agent_dispatcher/dispatcher.py @@ -275,12 +275,12 @@ async def create_process(self, executor: Executor, args): env = os.environ.copy() if isinstance(args, dict): for k in args: - env[k.upper()] = str(args[k]) + env[f"EXECUTOR_CONFIG_{k.upper()}"] = str(args[k]) else: logger.error("Args from data received has a not supported type") raise ValueError("Args from data received has a not supported type") for varenv, value in executor.varenvs.items(): - env[f"EXECUTOR_CONFIG_{varenv.upper()}"] = value + env[f"{varenv.upper()}"] = value process = await asyncio.create_subprocess_shell( executor.cmd, stdout=asyncio.subprocess.PIPE, From 1c2144dcff83b0c310756d986a30784a855cf01b Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Tue, 10 Dec 2019 13:51:56 -0300 Subject: [PATCH 39/41] [FIX] add_handler checks by handler name if its repeated and REPLACE it --- faraday_agent_dispatcher/logger.py | 5 +++++ tests/utils/testing_faraday_server.py | 1 + 2 files changed, 6 insertions(+) diff --git a/faraday_agent_dispatcher/logger.py b/faraday_agent_dispatcher/logger.py index 6ee35566..19da14a7 100644 --- a/faraday_agent_dispatcher/logger.py +++ b/faraday_agent_dispatcher/logger.py @@ -52,6 +52,7 @@ def setup_console_logging(formatter): console_handler = logging.StreamHandler() console_handler.setFormatter(formatter) console_handler.setLevel(config.LOGGING_LEVEL) + console_handler.name = "CONSOLE_HANDLER" add_handler(console_handler) LVL_SETTABLE_HANDLERS.append(console_handler) @@ -62,11 +63,15 @@ def setup_file_logging(formatter): log_file(), maxBytes=MAX_LOG_FILE_SIZE, backupCount=MAX_LOG_FILE_BACKUP_COUNT) file_handler.setFormatter(formatter) file_handler.setLevel(logging.DEBUG) + file_handler.name = "FILE_HANDLER" add_handler(file_handler) def add_handler(handler): logger = logging.getLogger(ROOT_LOGGER) + for hldr in logger.handlers: + if hldr.name == handler.name: + logger.removeHandler(hldr) logger.addHandler(handler) LOGGING_HANDLERS.append(handler) diff --git a/tests/utils/testing_faraday_server.py b/tests/utils/testing_faraday_server.py index f9dbe40b..0663ce0f 100644 --- a/tests/utils/testing_faraday_server.py +++ b/tests/utils/testing_faraday_server.py @@ -166,6 +166,7 @@ class TestLoggerHandler(StreamHandler): def __init__(self): super().__init__() self.history = [] + self.name = "TEST_HANDLER" def emit(self, record): self.history.append(record) From c8a71ded1c9418bec23716d50b72d056f9ca8567 Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Mon, 16 Dec 2019 11:45:20 -0300 Subject: [PATCH 40/41] [UPD] Integration test with faraday server, now with args --- tests/integration/faraday/test_execution.py | 29 ++++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/tests/integration/faraday/test_execution.py b/tests/integration/faraday/test_execution.py index eb7257cc..81db8042 100644 --- a/tests/integration/faraday/test_execution.py +++ b/tests/integration/faraday/test_execution.py @@ -18,6 +18,7 @@ WS_PORT = config.get(Sections.SERVER, "websocket_port") WORKSPACE = fuzzy_string(6).lower() # TODO FIX WHEN FARADAY ACCEPTS CAPITAL FIRST LETTER AGENT_NAME = fuzzy_string(6) +EXECUTOR_NAME = fuzzy_string(6) USER = os.getenv("FARADAY_USER") EMAIL = os.getenv("FARADAY_EMAIL") @@ -55,16 +56,28 @@ def test_execute_agent(): config.set(Sections.TOKENS, "registration", token) config.remove_option(Sections.TOKENS, "agent") config.set(Sections.SERVER, "workspace", WORKSPACE) - config.set(Sections.EXECUTOR, "agent_name", AGENT_NAME) + config.set(Sections.AGENT, "agent_name", AGENT_NAME) + config.set(Sections.AGENT, "executors", EXECUTOR_NAME) path_to_basic_executor = ( Path(__file__).parent.parent.parent / 'data' / 'basic_executor.py' ) + executor_section = Sections.EXECUTOR_DATA.format(EXECUTOR_NAME) + params_section = Sections.EXECUTOR_PARAMS.format(EXECUTOR_NAME) + for section in [executor_section, params_section]: + if section not in config: + config.add_section(section) + config.set( - Sections.EXECUTOR, + Sections.EXECUTOR_DATA.format(EXECUTOR_NAME), "cmd", - f"python {path_to_basic_executor} --out json" + f"python {path_to_basic_executor}" ) + + config.set(params_section, "out", "True") + [config.set(params_section, param, "False") for param in [ + "count", "spare", "spaced_before", "spaced_middle", "err", "fails"]] + save_config(CONFIG_DIR) # Init dispatcher! @@ -78,6 +91,7 @@ def test_execute_agent(): res_data = res.json() assert len(res_data) == 1, p.communicate(timeout=0.1) agent = res_data[0] + agent_id = agent["id"] if agent_ok_status_keys_set != set(agent.keys()): print("Keys set from agent endpoint differ from expected ones, checking if its a superset") assert agent_ok_status_keys_set.issubset(set(agent.keys())) @@ -86,7 +100,14 @@ def test_execute_agent(): # Run executor! res = session.post(api_url(HOST, API_PORT, postfix=f'/_api/v2/ws/{WORKSPACE}/agents/{agent["id"]}/run/'), - data={'csrf_token': session_res.json()['csrf_token']}) + json={ + 'csrf_token': session_res.json()['csrf_token'], + 'executorData': { + "agent_id": agent_id, + "executor": EXECUTOR_NAME, + "args": {"out": "json"} + } + }) assert res.status_code == 200, res.text time.sleep(2) # If fails check time From 854a2831d11675a09771e6d886be06fba8048a32 Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Tue, 17 Dec 2019 13:38:31 -0300 Subject: [PATCH 41/41] [UPD] faraday_agent_dispatcher references and version --- CONTRIBUTING.rst | 24 ++++++++++++------------ Makefile | 8 ++++---- docs/Makefile | 2 +- docs/conf.py | 20 ++++++++++---------- docs/index.rst | 2 +- docs/installation.rst | 16 ++++++++-------- docs/make.bat | 2 +- docs/usage.rst | 5 +++-- faraday_agent_dispatcher/__init__.py | 4 ++-- faraday_agent_dispatcher/cli.py | 2 +- setup.cfg | 4 ++-- setup.py | 2 +- tests/unittests/test_agent_dispatcher.py | 2 +- tox.ini | 2 +- 14 files changed, 48 insertions(+), 47 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 5778337f..a021dffd 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -15,7 +15,7 @@ Types of Contributions Report Bugs ~~~~~~~~~~~ -Report bugs at https://github.com/EricHorvat/dummy_faraday_agent/issues. +Report bugs at https://github.com/faradaysec/faraday_agent_dispatcher/issues. If you are reporting a bug, please include: @@ -38,14 +38,14 @@ and "help wanted" is open to whoever wants to implement it. Write Documentation ~~~~~~~~~~~~~~~~~~~ -dummy_faraday_agent could always use more documentation, whether as part of the -official dummy_faraday_agent docs, in docstrings, or even on the web in blog posts, +faraday_agent_dispatcher could always use more documentation, whether as part of the +official faraday_agent_dispatcher docs, in docstrings, or even on the web in blog posts, articles, and such. Submit Feedback ~~~~~~~~~~~~~~~ -The best way to send feedback is to file an issue at https://github.com/EricHorvat/dummy_faraday_agent/issues. +The best way to send feedback is to file an issue at https://github.com/faraday/faraday_agent_dispatcher/issues. If you are proposing a feature: @@ -57,17 +57,17 @@ If you are proposing a feature: Get Started! ------------ -Ready to contribute? Here's how to set up `dummy_faraday_agent` for local development. +Ready to contribute? Here's how to set up `faraday_agent_dispatcher` for local development. -1. Fork the `dummy_faraday_agent` repo on GitHub. +1. Fork the `faraday_agent_dispatcher` repo on GitHub. 2. Clone your fork locally:: - $ git clone git@github.com:your_name_here/dummy_faraday_agent.git + $ git clone git@github.com:your_name_here/faraday_agent_dispatcher.git 3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development:: - $ mkvirtualenv dummy_faraday_agent - $ cd dummy_faraday_agent/ + $ mkvirtualenv faraday_agent_dispatcher + $ cd faraday_agent_dispatcher/ $ python setup.py develop 4. Create a branch for local development:: @@ -79,7 +79,7 @@ Ready to contribute? Here's how to set up `dummy_faraday_agent` for local develo 5. When you're done making changes, check that your changes pass flake8 and the tests, including testing other Python versions with tox:: - $ flake8 dummy_faraday_agent tests + $ flake8 faraday_agent_dispatcher tests $ python setup.py test or py.test $ tox @@ -103,7 +103,7 @@ Before you submit a pull request, check that it meets these guidelines: your new functionality into a function with a docstring, and add the feature to the list in README.rst. 3. The pull request should work for Python 2.7, 3.4, 3.5 and 3.6, and for PyPy. Check - https://travis-ci.org/EricHorvat/dummy_faraday_agent/pull_requests + https://travis-ci.org/faradaysec/faraday_agent_dispatcher/pull_requests and make sure that the tests pass for all supported Python versions. Tips @@ -111,7 +111,7 @@ Tips To run a subset of tests:: -$ py.test tests.test_dummy_faraday_agent +$ py.test tests.faraday_agent_dispatcher Deploying diff --git a/Makefile b/Makefile index ea46ec5c..1399b142 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,7 @@ clean-test: ## remove test and coverage artifacts rm -fr .pytest_cache lint: ## check style with flake8 - flake8 dummy_faraday_agent tests + flake8 faraday_agent_dispatcher tests test: ## run tests quickly with the default Python py.test @@ -60,15 +60,15 @@ test-all: ## run tests on every Python version with tox tox coverage: ## check code coverage quickly with the default Python - coverage run --source dummy_faraday_agent -m pytest + coverage run --source faraday_agent_dispatcher -m pytest coverage report -m coverage html $(BROWSER) htmlcov/index.html docs: ## generate Sphinx HTML documentation, including API docs - rm -f docs/dummy_faraday_agent.rst + rm -f docs/faraday_agent_dispatcher.rst rm -f docs/modules.rst - sphinx-apidoc -o docs/ dummy_faraday_agent + sphinx-apidoc -o docs/ faraday_agent_dispatcher $(MAKE) -C docs clean $(MAKE) -C docs html $(BROWSER) docs/_build/html/index.html diff --git a/docs/Makefile b/docs/Makefile index e85ac610..e9afa1f1 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -4,7 +4,7 @@ # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = python -msphinx -SPHINXPROJ = dummy_faraday_agent +SPHINXPROJ = faraday_agent_dispatcher SOURCEDIR = . BUILDDIR = _build diff --git a/docs/conf.py b/docs/conf.py index 8a74d2ec..c363ce0a 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # -# faraday_dummy_agent documentation build configuration file, created by +# faraday_agent_dispatcher documentation build configuration file, created by # sphinx-quickstart on Fri Jun 9 13:47:02 2017. # # This file is execfile()d with the current directory set to its @@ -47,7 +47,7 @@ master_doc = 'index' # General information about the project. -project = u'faraday_dummy_agent' +project = u'faraday_agent_dispatcher' copyright = u"2019, Eric Horvat" author = u"Eric Horvat" @@ -101,7 +101,7 @@ # -- Options for HTMLHelp output --------------------------------------- # Output file base name for HTML help builder. -htmlhelp_basename = 'dummy_faraday_agentdoc' +htmlhelp_basename = 'faraday_agent_dispatcher' # -- Options for LaTeX output ------------------------------------------ @@ -128,8 +128,8 @@ # (source start file, target name, title, author, documentclass # [howto, manual, or own class]). latex_documents = [ - (master_doc, 'faraday_dummy_agent.tex', - u'faraday_dummy_agent Documentation', + (master_doc, 'faraday_agent_dispatcher.tex', + u'faraday_agent_dispatcher Documentation', u'Eric Horvat', 'manual'), ] @@ -139,8 +139,8 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - (master_doc, 'faraday_dummy_agent', - u'faraday_dummy_agent Documentation', + (master_doc, 'faraday_agent_dispatcher', + u'faraday_agent_dispatcher Documentation', [author], 1) ] @@ -151,10 +151,10 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'faraday_dummy_agent', - u'faraday_dummy_agent Documentation', + (master_doc, 'faraday_agent_dispatcher', + u'faraday_agent_dispatcher Documentation', author, - 'faraday_dummy_agent', + 'faraday_agent_dispatcher', 'One line description of project.', 'Miscellaneous'), ] diff --git a/docs/index.rst b/docs/index.rst index 4d349063..abceba8c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,4 +1,4 @@ -Welcome to dummy_faraday_agent's documentation! +Welcome to faraday_agent_dispatcher's documentation! ====================================== .. toctree:: diff --git a/docs/installation.rst b/docs/installation.rst index c1cc67e0..c9ed0526 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -8,13 +8,13 @@ Installation Stable release -------------- -To install dummy_faraday_agent, run this command in your terminal: +To install faraday_agent_dispatcher, run this command in your terminal: .. code-block:: console - $ pip install dummy_faraday_agent + $ pip install faraday_agent_dispatcher -This is the preferred method to install dummy_faraday_agent, as it will always install the most recent stable release. +This is the preferred method to install faraday_agent_dispatcher, as it will always install the most recent stable release. If you don't have `pip`_ installed, this `Python installation guide`_ can guide you through the process. @@ -26,19 +26,19 @@ you through the process. From sources ------------ -The sources for dummy_faraday_agent can be downloaded from the `Github repo`_. +The sources for faraday_agent_dispatcher can be downloaded from the `Github repo`_. You can either clone the public repository: .. code-block:: console - $ git clone git://github.com/EricHorvat/dummy_faraday_agent + $ git clone git://github.com/faradaysec/faraday_agent_dispatcher Or download the `tarball`_: .. code-block:: console - $ curl -OL https://github.com/EricHorvat/dummy_faraday_agent/tarball/master + $ curl -OL https://github.com/faradaysec/faraday_agent_dispatcher/tarball/master Once you have a copy of the source, you can install it with: @@ -47,5 +47,5 @@ Once you have a copy of the source, you can install it with: $ python setup.py install -.. _Github repo: https://github.com/EricHorvat/dummy_faraday_agent -.. _tarball: https://github.com/EricHorvat/dummy_faraday_agent/tarball/master +.. _Github repo: https://github.com/faradaysec/faraday_agent_dispatcher +.. _tarball: https://github.com/faradaysec/faraday_agent_dispatcher/tarball/master diff --git a/docs/make.bat b/docs/make.bat index 66e06c11..9c122b2c 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -9,7 +9,7 @@ if "%SPHINXBUILD%" == "" ( ) set SOURCEDIR=. set BUILDDIR=_build -set SPHINXPROJ=dummy_faraday_agent +set SPHINXPROJ=faraday_agent_dispatcher if "%1" == "" goto help diff --git a/docs/usage.rst b/docs/usage.rst index f79ec90e..dd67e1b5 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -2,6 +2,7 @@ Usage ===== -To use dummy_faraday_agent in a project:: +To use faraday_agent_dispatcher in a project:: + + import faraday_agent_dispatcher - import dummy_faraday_agent diff --git a/faraday_agent_dispatcher/__init__.py b/faraday_agent_dispatcher/__init__.py index 6ab7ab62..50a19dab 100644 --- a/faraday_agent_dispatcher/__init__.py +++ b/faraday_agent_dispatcher/__init__.py @@ -16,8 +16,8 @@ # along with this program. If not, see . -"""Top-level package for faraday_dummy_agent.""" +"""Top-level package for faraday_agent_dispatcher.""" __author__ = """Eric Horvat""" __email__ = 'erich@infobytesec.com' -__version__ = '0.1.0' +__version__ = '1.0' diff --git a/faraday_agent_dispatcher/cli.py b/faraday_agent_dispatcher/cli.py index a5e65e85..3fbad141 100644 --- a/faraday_agent_dispatcher/cli.py +++ b/faraday_agent_dispatcher/cli.py @@ -15,7 +15,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -"""Console script for faraday_dummy_agent.""" +"""Console script for faraday_agent_dispatcher.""" import os import sys import shutil diff --git a/setup.cfg b/setup.cfg index 2c38743e..1ce9b3c0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.1.0 +current_version = 1.0.0 commit = True tag = True @@ -7,7 +7,7 @@ tag = True search = version='{current_version}' replace = version='{new_version}' -[bumpversion:file:dummy_faraday_agent/__init__.py] +[bumpversion:file:faraday_agent_dispatcher/__init__.py] search = __version__ = '{current_version}' replace = __version__ = '{new_version}' diff --git a/setup.py b/setup.py index 11b33cbf..c5c55d65 100644 --- a/setup.py +++ b/setup.py @@ -70,6 +70,6 @@ test_suite='tests', tests_require=test_requirements, url='https://github.com/infobyte/faraday_agent_dispatcher', - version='0.1', + version='1.0', zip_safe=False, ) diff --git a/tests/unittests/test_agent_dispatcher.py b/tests/unittests/test_agent_dispatcher.py index 192a06b1..eba89364 100644 --- a/tests/unittests/test_agent_dispatcher.py +++ b/tests/unittests/test_agent_dispatcher.py @@ -16,7 +16,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -"""Tests for `faraday_dummy_agent` package.""" +"""Tests for `faraday_agent_dispatcher` package.""" import json import os diff --git a/tox.ini b/tox.ini index ba9723ce..0709d924 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ python = [testenv:flake8] basepython = python deps = flake8 -commands = flake8 dummy_faraday_agent +commands = flake8 faraday_agent_dispatcher [testenv] setenv =