[bug] adding list serialization for sets to messages sent to sqs (#867)

* adding list serialization for sets to messages sent to sqs

* addressing pr feedback and fixing test cases

* bumping version to 2.1.4

stream_alert/__init__.py

@@ -1,2 +1,2 @@
 """StreamAlert version."""
-__version__ = '2.1.3'
+__version__ = '2.1.4'

stream_alert/alert_merger/main.py

@@ -120,8 +120,8 @@ def _dispatch_alert(self, alert):
         LOGGER.info('Dispatching %s to %s (attempt %d)', alert, self.alert_proc, alert.attempts)
         MetricLogger.log_metric(ALERT_MERGER_NAME, MetricLogger.ALERT_ATTEMPTS, alert.attempts)
 
-        record_payload = json.dumps(
-            alert.dynamo_record(), cls=Alert.Encoder, separators=(',', ':'))
+        record_payload = json.dumps(alert.dynamo_record(), default=list, separators=(',', ':'))
+
         if len(record_payload) <= self.MAX_LAMBDA_PAYLOAD_SIZE:
             # The entire alert fits in the Lambda payload - send it all
             payload = record_payload

stream_alert/shared/alert.py

@@ -27,13 +27,6 @@ class AlertCreationError(Exception):
 class Alert(object):
     """Encapsulates a single alert and handles serializing to Dynamo and merging."""
 
-    class Encoder(json.JSONEncoder):
-        """Custom JSON encoder which handles sets."""
-        def default(self, obj):  # pylint: disable=arguments-differ,method-hidden
-            if isinstance(obj, set):
-                return list(obj)
-            return json.JSONEncoder.default(self, obj)
-
     _EXPECTED_INIT_KWARGS = {
         'alert_id', 'attempts', 'cluster', 'context', 'created', 'dispatched', 'log_source',
         'log_type', 'merge_by_keys', 'merge_window', 'outputs_sent', 'rule_description',
@@ -110,7 +103,7 @@ def __lt__(self, other):
 
     def __repr__(self):
         """Complete representation (for debugging) is an indented JSON string with all fields."""
-        return json.dumps(self.dynamo_record(), cls=self.Encoder, indent=2, sort_keys=True)
+        return json.dumps(self.dynamo_record(), default=list, indent=2, sort_keys=True)
 
     def __str__(self):
         """Simple string representation includes alert ID and triggered rule."""
@@ -158,7 +151,7 @@ def dynamo_record(self):
             'OutputsSent': self.outputs_sent or None,  # Empty sets not allowed by Dynamo
             # Compact JSON encoding (no spaces). We have to JSON-encode here
             # (instead of just passing the dict) because Dynamo does not allow empty string values.
-            'Record': json.dumps(self.record, separators=(',', ':'), cls=self.Encoder),
+            'Record': json.dumps(self.record, separators=(',', ':'), default=list),
             'RuleDescription': self.rule_description,
             'SourceEntity': self.source_entity,
             'SourceService': self.source_service,

stream_alert/shared/normalize.py

@@ -58,7 +58,7 @@ def match_types(cls, record, normalized_types):
             }
         """
         return {
-            key: set(cls._extract_values(record, set(keys_to_normalize)))
+            key: sorted(set(cls._extract_values(record, set(keys_to_normalize))))
             for key, keys_to_normalize in normalized_types.iteritems()
         }
 

tests/unit/stream_alert_shared/test_alert.py

@@ -62,7 +62,7 @@ def _customized_alert():
 
     def test_alert_encoder_invalid_json(self):
         """Alert Class - Alert Encoder - Invalid JSON raises parent exception"""
-        assert_raises(TypeError, json.dumps, RuntimeWarning, cls=Alert.Encoder)
+        assert_raises(TypeError, json.dumps, RuntimeWarning, default=list)
 
     def test_init_invalid_kwargs(self):
         """Alert Class - Init With Invalid Kwargs"""

tests/unit/stream_alert_shared/test_normalizer.py

@@ -50,9 +50,9 @@ def test_match_types(self):
             'ipv4': ['destination', 'source', 'sourceIPAddress']
         }
         expected_results = {
-            'sourceAccount': {123456},
-            'ipv4': {'1.1.1.2', '1.1.1.3'},
-            'region': {'region_name'}
+            'sourceAccount': [123456],
+            'ipv4': ['1.1.1.2', '1.1.1.3'],
+            'region': ['region_name']
         }
 
         results = Normalizer.match_types(self._test_record(), normalized_types)
@@ -67,10 +67,10 @@ def test_match_types_multiple(self):
             'userName': ['userName', 'owner', 'invokedBy']
         }
         expected_results = {
-            'account': {123456},
-            'ipv4': {'1.1.1.2', '1.1.1.3'},
-            'region': {'region_name'},
-            'userName': {'Alice', 'signin.amazonaws.com'}
+            'account': [123456],
+            'ipv4': ['1.1.1.2', '1.1.1.3'],
+            'region': ['region_name'],
+            'userName': ['Alice', 'signin.amazonaws.com']
         }
 
         results = Normalizer.match_types(self._test_record(), normalized_types)
@@ -82,7 +82,7 @@ def test_match_types_list(self):
             'ipv4': ['sourceIPAddress'],
         }
         expected_results = {
-            'ipv4': {'1.1.1.2', '1.1.1.3'}
+            'ipv4': ['1.1.1.2', '1.1.1.3']
         }
 
         test_record = {
@@ -124,8 +124,8 @@ def test_normalize(self):
             },
             'sourceIPAddress': '1.1.1.3',
             'streamalert:normalization': {
-                'region': {'region_name'},
-                'sourceAccount': {123456}
+                'region': ['region_name'],
+                'sourceAccount': [123456]
             }
         }
 
@@ -164,7 +164,7 @@ def test_normalize_bad_normalized_key(self):
             },
             'sourceIPAddress': '1.1.1.3',
             'streamalert:normalization': {
-                'bad_type': set(),
+                'bad_type': list(),
             }
         }