From 842949948dc131f1bc13557482d9cd6eb3edfa01 Mon Sep 17 00:00:00 2001 From: Jonathan Kovarik Date: Thu, 6 Jan 2022 20:10:23 -0700 Subject: [PATCH 1/7] Update cumulus-message-adapter such that event.task_config isn't required --- CHANGELOG.md | 7 +++ message_adapter/cumulus_message.py | 2 +- message_adapter/message_adapter.py | 4 +- tests/test-program.py | 81 ++++++++++++++++++------------ 4 files changed, 58 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0dcbfb1..464027c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ All notable changes to this project are documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [v2.0.1] 2022-01-07 + +### Fixed + +- **CUMULUS-2751** + - Fix regression in 2.0.0 that caused tasks without `task_config` defined cause CMA to fail + ## [v2.0.0] 2021-12-17 ### BREAKING CHANGES diff --git a/message_adapter/cumulus_message.py b/message_adapter/cumulus_message.py index 1c8bdca..3859bca 100644 --- a/message_adapter/cumulus_message.py +++ b/message_adapter/cumulus_message.py @@ -20,7 +20,7 @@ def load_config(event): if 'task_config' in event: return event['task_config'] - return None + return {} def load_remote_event(event): """ diff --git a/message_adapter/message_adapter.py b/message_adapter/message_adapter.py index e92f5e1..187bf6b 100644 --- a/message_adapter/message_adapter.py +++ b/message_adapter/message_adapter.py @@ -100,8 +100,8 @@ def load_nested_event(self, event): final_payload = resolve_input(event, config) response = {'input': final_payload} self.__validate_json(final_payload, 'input') - self.__validate_json(final_config, 'config') - if final_config is not None: + if final_config: + self.__validate_json(final_config, 'config') response['config'] = final_config if 'cumulus_message' in config: response['messageConfig'] = config['cumulus_message'] diff --git a/tests/test-program.py b/tests/test-program.py index 1f3ece7..3708295 100644 --- a/tests/test-program.py +++ b/tests/test-program.py @@ -84,23 +84,26 @@ def write_streaming_input(self, command, proc_input, p_stdin): p_stdin.write('\n'.encode('utf-8')) p_stdin.flush() - def transform_messages_streaming(self, testcase, context=None): + def transform_messages_streaming(self, params): """ Given a testcase, run 'streaming' interface against input and check if outputs are correct """ - if context is None: - context = {} - inp = open(os.path.join(self.test_folder, f'{testcase}.input.json'), encoding='utf-8') - in_msg = json.loads(inp.read()) - s3meta = None - if 'replace' in in_msg: - s3meta = self.place_remote_message(in_msg) schemas = { 'input': 'schemas/examples-messages.output.json', 'output': 'schemas/examples-messages.output.json', 'config': 'schemas/examples-messages.config.json' } + context = params.get('context', {}) + schemas = params.get('schemas', schemas) + testcase = params['testcase'] + + inp = open(os.path.join(self.test_folder, f'{testcase}.input.json'), encoding='utf-8') + in_msg = json.loads(inp.read()) + s3meta = None + if 'replace' in in_msg: + s3meta = self.place_remote_message(in_msg) + cma_input = {'event': in_msg, 'context': context, 'schemas': schemas} current_directory = os.getcwd() @@ -135,24 +138,27 @@ def transform_messages_streaming(self, testcase, context=None): if s3meta is not None: self.clean_up_remote_message(s3meta['bucket_name'], s3meta['key_name']) - def transform_messages(self, testcase, context=None): + def transform_messages(self, params): """ transform cumulus messages, and check if the command return status and outputs are correct. Each test case (such as 'basic') has its corresponding example messages and schemas. """ - if context is None: - context = {} - inp = open(os.path.join(self.test_folder, f'{testcase}.input.json'), encoding='utf-8') - in_msg = json.loads(inp.read()) - s3meta = None - if 'replace' in in_msg: - s3meta = self.place_remote_message(in_msg) schemas = { 'input': 'schemas/examples-messages.output.json', 'output': 'schemas/examples-messages.output.json', 'config': 'schemas/examples-messages.config.json' } + context = params.get('context', None) + schemas = params.get('schemas', schemas) + testcase = params['testcase'] + + inp = open(os.path.join(self.test_folder, f'{testcase}.input.json'), encoding='utf-8') + in_msg = json.loads(inp.read()) + s3meta = None + if 'replace' in in_msg: + s3meta = self.place_remote_message(in_msg) + all_input = {'event': in_msg, 'context': context, 'schemas': schemas} current_directory = os.getcwd() @@ -194,38 +200,47 @@ def transform_messages(self, testcase, context=None): def test_basic(self): """ test basic message """ - self.transform_messages('basic') - self.transform_messages_streaming('basic') + self.transform_messages({ 'testcase': 'basic'}) + self.transform_messages_streaming({ 'testcase': 'basic'}) + + def test_basic_no_config(self): + """ test basic no config message """ + schemas = { + 'input': 'schemas/examples-messages.output.json', + 'output': 'schemas/examples-messages-no-config.output.json', + } + self.transform_messages({ 'testcase': 'basic_no_config', 'schemas': schemas }) + self.transform_messages_streaming({ 'testcase': 'basic_no_config'}) def test_exception(self): """ test remote message with exception """ - self.transform_messages('exception') - self.transform_messages_streaming('exception') + self.transform_messages({ 'testcase': 'exception'}) + self.transform_messages_streaming({ 'testcase': 'exception'}) def test_jsonpath(self): """ test jsonpath message """ - self.transform_messages('jsonpath') - self.transform_messages_streaming('jsonpath') + self.transform_messages({ 'testcase': 'jsonpath'}) + self.transform_messages_streaming({ 'testcase': 'jsonpath'}) def test_meta(self): """ test meta message """ - self.transform_messages('meta') - self.transform_messages_streaming('meta') + self.transform_messages({ 'testcase': 'meta'}) + self.transform_messages_streaming({ 'testcase': 'meta'}) def test_remote(self): """ test remote message """ - self.transform_messages('remote') - self.transform_messages_streaming('remote') + self.transform_messages({ 'testcase': 'remote'}) + self.transform_messages_streaming({ 'testcase': 'remote'}) def test_templates(self): """ test templates message """ - self.transform_messages('templates') - self.transform_messages_streaming('templates') + self.transform_messages({ 'testcase': 'templates'}) + self.transform_messages_streaming({ 'testcase': 'templates'}) def test_validation_failure_case(self): """ test validation failure case """ try: - self.transform_messages("invalidinput") + self.transform_messages({ 'testcase': 'invalidinput'}) except AssertionError: return assert False @@ -237,8 +252,8 @@ def test_workflow_task_meta(self): 'invokedFunctionArn': 'fakearn', 'functionVersion': '1', } - self.transform_messages('workflow_tasks', context) - self.transform_messages_streaming('workflow_tasks', context) + self.transform_messages({'testcase': 'workflow_tasks', 'context': context}) + self.transform_messages_streaming({'testcase': 'workflow_tasks', 'context': context}) def test_multiple_workflow_tasks_meta(self): """ test multiple meta.workflow_task entries""" @@ -247,5 +262,5 @@ def test_multiple_workflow_tasks_meta(self): 'invokedFunctionArn': 'fakearn2', 'functionVersion': '2', } - self.transform_messages('workflow_tasks_multiple', context) - self.transform_messages_streaming('workflow_tasks_multiple', context) + self.transform_messages({'testcase': 'workflow_tasks_multiple', 'context': context}) + self.transform_messages_streaming({'testcase': 'workflow_tasks_multiple', 'context': context}) From 6e7caf5856bb58807cc71b66ed5f65420abb8fff Mon Sep 17 00:00:00 2001 From: Jonathan Kovarik Date: Thu, 6 Jan 2022 21:04:39 -0700 Subject: [PATCH 2/7] Version up --- message_adapter/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_adapter/version.py b/message_adapter/version.py index 4bd7cbd..c8fda1b 100644 --- a/message_adapter/version.py +++ b/message_adapter/version.py @@ -1 +1 @@ -__version__ = 'v2.0.0' +__version__ = 'v2.0.1' From b7497308ae7c78003e908954e2d331eece733d24 Mon Sep 17 00:00:00 2001 From: Jonathan Kovarik Date: Fri, 7 Jan 2022 06:28:23 -0700 Subject: [PATCH 3/7] Lint :bell: --- tests/test-program.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test-program.py b/tests/test-program.py index 3708295..eb87e12 100644 --- a/tests/test-program.py +++ b/tests/test-program.py @@ -263,4 +263,5 @@ def test_multiple_workflow_tasks_meta(self): 'functionVersion': '2', } self.transform_messages({'testcase': 'workflow_tasks_multiple', 'context': context}) - self.transform_messages_streaming({'testcase': 'workflow_tasks_multiple', 'context': context}) + self.transform_messages_streaming( + {'testcase': 'workflow_tasks_multiple', 'context': context}) From 5e8342f75cb6122921c7f5b95dea464b7c11424f Mon Sep 17 00:00:00 2001 From: Jonathan Kovarik Date: Fri, 7 Jan 2022 07:19:52 -0700 Subject: [PATCH 4/7] Add missing schemas --- examples/messages/basic_no_config.input.json | 12 ++++++++++++ examples/messages/basic_no_config.output.json | 13 +++++++++++++ .../examples-messages-no-config.output.json | 15 +++++++++++++++ 3 files changed, 40 insertions(+) create mode 100644 examples/messages/basic_no_config.input.json create mode 100644 examples/messages/basic_no_config.output.json create mode 100644 examples/schemas/examples-messages-no-config.output.json diff --git a/examples/messages/basic_no_config.input.json b/examples/messages/basic_no_config.input.json new file mode 100644 index 0000000..03492e6 --- /dev/null +++ b/examples/messages/basic_no_config.input.json @@ -0,0 +1,12 @@ +{ + "cumulus_meta": { + "message_source": "local", + "id": "id-1234" + }, + "meta": { + "foo": "bar" + }, + "payload": { + "anykey": "anyvalue" + } +} diff --git a/examples/messages/basic_no_config.output.json b/examples/messages/basic_no_config.output.json new file mode 100644 index 0000000..149a0d0 --- /dev/null +++ b/examples/messages/basic_no_config.output.json @@ -0,0 +1,13 @@ +{ + "cumulus_meta": { + "message_source": "local", + "id": "id-1234" + }, + "meta": { + "foo": "bar" + }, + "payload": { + "input": { "anykey": "anyvalue" } + }, + "exception": "None" + } diff --git a/examples/schemas/examples-messages-no-config.output.json b/examples/schemas/examples-messages-no-config.output.json new file mode 100644 index 0000000..3899b18 --- /dev/null +++ b/examples/schemas/examples-messages-no-config.output.json @@ -0,0 +1,15 @@ +{ + "properties": { + "input": { + "type": "object" + }, + "config": { + "type": "object" + }, + "anykey": { + "type": "string" + } + }, + "additionalProperties": false + } + From 7620bac9cd8b7aba2a6d4d01366156474e072023 Mon Sep 17 00:00:00 2001 From: Jonathan Kovarik Date: Fri, 7 Jan 2022 08:58:30 -0700 Subject: [PATCH 5/7] Respond to PR feedback --- tests/test-program.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test-program.py b/tests/test-program.py index eb87e12..f328943 100644 --- a/tests/test-program.py +++ b/tests/test-program.py @@ -90,7 +90,7 @@ def transform_messages_streaming(self, params): """ schemas = { - 'input': 'schemas/examples-messages.output.json', + 'input': 'schemas/examples-messages.input.json', 'output': 'schemas/examples-messages.output.json', 'config': 'schemas/examples-messages.config.json' } @@ -145,7 +145,7 @@ def transform_messages(self, params): """ schemas = { - 'input': 'schemas/examples-messages.output.json', + 'input': 'schemas/examples-messages.input.json', 'output': 'schemas/examples-messages.output.json', 'config': 'schemas/examples-messages.config.json' } @@ -206,7 +206,7 @@ def test_basic(self): def test_basic_no_config(self): """ test basic no config message """ schemas = { - 'input': 'schemas/examples-messages.output.json', + 'input': 'schemas/examples-messages.input.json', 'output': 'schemas/examples-messages-no-config.output.json', } self.transform_messages({ 'testcase': 'basic_no_config', 'schemas': schemas }) From 5aae03d5fc1a560851419f5b1fd95cdee381b633 Mon Sep 17 00:00:00 2001 From: Jonathan Kovarik Date: Fri, 7 Jan 2022 09:01:56 -0700 Subject: [PATCH 6/7] Update fixture :bell: --- examples/schemas/examples-messages-no-config.output.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/examples/schemas/examples-messages-no-config.output.json b/examples/schemas/examples-messages-no-config.output.json index 3899b18..7909b01 100644 --- a/examples/schemas/examples-messages-no-config.output.json +++ b/examples/schemas/examples-messages-no-config.output.json @@ -3,9 +3,6 @@ "input": { "type": "object" }, - "config": { - "type": "object" - }, "anykey": { "type": "string" } From 064847b50c3b7d31454ec78e5dfec95a1ec11ee8 Mon Sep 17 00:00:00 2001 From: Jonathan Kovarik Date: Fri, 7 Jan 2022 09:13:34 -0700 Subject: [PATCH 7/7] Update test to properly pass schemas, more tightly constrain test case schema --- examples/schemas/examples-messages-no-config.output.json | 3 --- tests/test-program.py | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/examples/schemas/examples-messages-no-config.output.json b/examples/schemas/examples-messages-no-config.output.json index 7909b01..d8dea83 100644 --- a/examples/schemas/examples-messages-no-config.output.json +++ b/examples/schemas/examples-messages-no-config.output.json @@ -2,9 +2,6 @@ "properties": { "input": { "type": "object" - }, - "anykey": { - "type": "string" } }, "additionalProperties": false diff --git a/tests/test-program.py b/tests/test-program.py index f328943..83ce201 100644 --- a/tests/test-program.py +++ b/tests/test-program.py @@ -210,7 +210,7 @@ def test_basic_no_config(self): 'output': 'schemas/examples-messages-no-config.output.json', } self.transform_messages({ 'testcase': 'basic_no_config', 'schemas': schemas }) - self.transform_messages_streaming({ 'testcase': 'basic_no_config'}) + self.transform_messages_streaming({ 'testcase': 'basic_no_config', 'schemas': schemas }) def test_exception(self): """ test remote message with exception """