diff --git a/CHANGELOG.md b/CHANGELOG.md index e252c75..52e92bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project are documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [v1.1.3] + +### Fixed + +- Removed unsafe use of `exec` and `eval` in `message_adapter.__assignJsonPathValue` + ## [v1.1.2] ### Added diff --git a/README.md b/README.md index 9da64a5..d3628fe 100644 --- a/README.md +++ b/README.md @@ -6,19 +6,20 @@ Read more about how the `cumulus-message-adapter` works in the [CONTRACT.md](./CONTRACT.md). - ## Releases ### Release Versions -Please note the following convention for release versions: -X.Y.Z: where: +Please note the following convention for release versions: + +X.Y.Z: where: * X is an organizational release that signifies the completion of a core set of functionality * Y is a major version release that may include incompatible API changes and/or other breaking changes * Z is a minor version that includes bugfixes and backwards compatible improvements ### Continuous Integration + [CircleCI](https://circleci.com/gh/nasa/cumulus-message-adapter) manages releases and release assets. Whenever CircleCI passes on the master branch of cumulus-message-adapter and `message_adapter/version.py` has been updated with a version that doesn't match an existing tag, CircleCI will: @@ -33,8 +34,10 @@ These steps are fully detailed in the [`.circleci/config.yml`](./.circleci/confi ### Dependency Installation - $ pip install -r requirements-dev.txt - $ pip install -r requirements.txt +```shell +pip install -r requirements-dev.txt +pip install -r requirements.txt +``` ### Running Tests @@ -42,38 +45,38 @@ Running tests requires [localstack](https://github.com/localstack/localstack). Tests only require localstack running S3, which can be initiated with the following command: -``` -$ SERVICES=s3 localstack start +```shell +SERVICES=s3 localstack start ``` And then you can check tests pass with the following nosetests command: -``` -$ CUMULUS_ENV=testing nosetests -v -s +```shell +CUMULUS_ENV=testing nosetests -v -s ``` ### Linting - $ pylint message_adapter +```shell +pylint message_adapter +``` ### Contributing If changes are made to the codebase, you can create the cumulus-message-adapter zip archive for testing libraries that require it: -```bash -$ make clean -$ make cumulus-message-adapter.zip +```shell +make clean +make cumulus-message-adapter.zip ``` Then you can run some integration tests: -```bash -./examples/example-node-message-adapter-lib.js +```shell +./examples/example-node-message-adapter-lib.js ``` - ### Troubleshooting * Error: "DistutilsOptionError: must supply either home or prefix/exec-prefix — not both" when running `make cumulus-message-adapter.zip` * [Solution](https://stackoverflow.com/a/24357384) - diff --git a/message_adapter/message_adapter.py b/message_adapter/message_adapter.py index 3ac69d8..bf63ab9 100644 --- a/message_adapter/message_adapter.py +++ b/message_adapter/message_adapter.py @@ -1,18 +1,15 @@ import os import json import re -import warnings import sys +from copy import deepcopy from datetime import datetime, timedelta import uuid from jsonpath_ng import parse from jsonschema import validate -from collections import defaultdict -from copy import deepcopy from .aws import stepFn, s3 - class message_adapter: """ transforms the cumulus message @@ -227,7 +224,7 @@ def __validate_json(self, document, schema_type): raise e # Config templating - def __resolvePathStr(self, event, str): + def __resolvePathStr(self, event, jsonPathString): """ * Given a Cumulus message (AWS Lambda event) and a string containing a JSONPath * template to interpret, returns the result of interpreting that template. @@ -243,32 +240,31 @@ def __resolvePathStr(self, event, str): * It's likely we'll need some sort of bracket-escaping at some point down the line * * @param {*} event The Cumulus message - * @param {*} str A string containing a JSONPath template to resolve + * @param {*} jsonPathString A string containing a JSONPath template to resolve * @returns {*} The resolved object """ - valueRegex = '^{[^\[\]].*}$' - arrayRegex = '^{\[.*\]}$' + valueRegex = r"^{[^\[\]].*}$" + arrayRegex = r"^{\[.*\]}$" templateRegex = '{[^}]+}' - if (re.search(valueRegex, str)): - matchData = parse(str.lstrip('{').rstrip('}')).find(event) - return matchData[0].value if len(matchData) > 0 else None + if re.search(valueRegex, jsonPathString): + matchData = parse(jsonPathString.lstrip('{').rstrip('}')).find(event) + return matchData[0].value if matchData else None - elif (re.search(arrayRegex, str)): - matchData = parse(str.lstrip('{').rstrip('}').lstrip('[').rstrip(']')).find(event) - return [item.value for item in matchData] if len(matchData) > 0 else [] + elif re.search(arrayRegex, jsonPathString): + parsedJsonPath = jsonPathString.lstrip('{').rstrip('}').lstrip('[').rstrip(']'); + matchData = parse(parsedJsonPath).find(event) + return [item.value for item in matchData] if matchData else [] - elif (re.search(templateRegex, str)): - matches = re.findall(templateRegex, str) + elif re.search(templateRegex, jsonPathString): + matches = re.findall(templateRegex, jsonPathString) for match in matches: matchData = parse(match.lstrip('{').rstrip('}')).find(event) - if len(matchData) > 0: - str = str.replace(match, matchData[0].value) - return str + if matchData: + jsonPathString = jsonPathString.replace(match, matchData[0].value) + return jsonPathString - return str - - raise LookupError('Could not resolve path ' + str) + return jsonPathString def __resolveConfigObject(self, event, config): """ @@ -347,7 +343,7 @@ def loadNestedEvent(self, event, context): if finalConfig is not None: response['config'] = finalConfig if 'cumulus_message' in config: - response['messageConfig'] = config[ 'cumulus_message'] + response['messageConfig'] = config['cumulus_message'] # add cumulus_config property, only selective attributes from event.cumulus_meta are added if 'cumulus_meta' in event: @@ -379,21 +375,21 @@ def __assignJsonPathValue(self, message, jspath, value): * @param {*} message The message to be update * @return {*} updated message """ - if len(parse(jspath).find(message)) > 0: - parse(jspath).update(message, value) - else: + if not parse(jspath).find(message): paths = jspath.lstrip('$.').split('.') currentItem = message - dictPath = str() keyNotFound = False for path in paths: - dictPath += "['" + path + "']" if keyNotFound or path not in currentItem: keyNotFound = True - exec("message" + dictPath + " = {}") - currentItem = eval("message" + dictPath) - - exec("message" + dictPath + " = value") + newPathDict = {} + # Add missing key to existing dict + currentItem[path] = newPathDict + # Set current item to newly created dict + currentItem = newPathDict + else: + currentItem = currentItem[path] + parse(jspath).update(message, value) return message def __assignOutputs(self, handlerResponse, event, messageConfig): diff --git a/message_adapter/version.py b/message_adapter/version.py index 218644a..8f688c6 100644 --- a/message_adapter/version.py +++ b/message_adapter/version.py @@ -1 +1 @@ -__version__ = 'v1.1.2' \ No newline at end of file +__version__ = 'v1.1.3' diff --git a/tests/test-module.py b/tests/test-module.py index 4148733..0bc92ac 100644 --- a/tests/test-module.py +++ b/tests/test-module.py @@ -201,6 +201,31 @@ def test_result_payload_with_nested_config_outputs(self): self.nested_response, {}, message_config_with_nested_outputs) assert result['payload'] == {'dataLocation': 's3://source.jpg'} + def test_result_payload_with_nested_sibling_config_outputs(self): + """ + Test nested payload value is updated when messageConfig contains + outputs templates where sibling nodes exist + """ + message_config_with_nested_outputs = { + 'outputs': [{ + 'source': '{$.input.dataLocation}', + 'destination': '{$.test.dataLocation}' + }] + } + + event = { + 'test': { + 'key': 'value' + } + } + + result = self.cumulus_message_adapter._message_adapter__assignOutputs( # pylint: disable=no-member + self.nested_response, event, message_config_with_nested_outputs) + assert result['test'] == { + 'dataLocation': 's3://source.jpg', + 'key': 'value' + } + # createNextEvent tests def test_with_replace(self): """