Support min max object size in s3_lifecycle module (#2205)

SUMMARY

Support the S3 lifecycle settings of minimum and maximum object size to apply the lifecycle rules.

Fixes #861
ISSUE TYPE


Feature Pull Request

COMPONENT NAME

s3_lifecycle
ADDITIONAL INFORMATION

Reviewed-by: Mark Chappell
Reviewed-by: Javid Alimohideen

changelogs/fragments/2205-support-minmax-s3lifecycle.yml

@@ -0,0 +1,2 @@
+minor_changes:
+  - s3_lifecycle - Support for min and max object size when applying the filter rules (https://github.com/ansible-collections/community.aws/pull/2205).

plugins/modules/s3_lifecycle.py

@@ -59,6 +59,18 @@
         and replaced with the new transition(s)
     default: true
     type: bool
+  maximum_object_size:
+    description:
+      - The maximum object size to which the rule applies.
+    required: false
+    type: int
+    version_added: 9.1.0
+  minimum_object_size:
+    description:
+      - The minimum object size to which the rule applies.
+    required: false
+    version_added: 9.1.0
+    type: int
   noncurrent_version_expiration_days:
     description:
       - The number of days after which non-current versions should be deleted.
@@ -272,12 +284,31 @@ def fetch_rules(client, module, name):
     return current_lifecycle_rules
 
 
+# Helper function to deeply compare filters
+def filters_are_equal(filter1, filter2):
+    if filter1 == filter2:
+        return True
+    if not filter1 or not filter2:
+        return False
+    # Treat empty string as equal to a filter not being set
+    return (
+        filter1.get("Prefix", "") == filter2.get("Prefix", "")
+        and filter1.get("ObjectSizeGreaterThan") == filter2.get("ObjectSizeGreaterThan")
+        and filter1.get("ObjectSizeLessThan") == filter2.get("ObjectSizeLessThan")
+        and filter1.get("And", {}).get("Prefix", "") == filter2.get("And", {}).get("Prefix", "")
+        and filter1.get("And", {}).get("ObjectSizeGreaterThan") == filter2.get("And", {}).get("ObjectSizeGreaterThan")
+        and filter1.get("And", {}).get("ObjectSizeLessThan") == filter2.get("And", {}).get("ObjectSizeLessThan")
+    )
+
+
 def build_rule(client, module):
     name = module.params.get("name")
     abort_incomplete_multipart_upload_days = module.params.get("abort_incomplete_multipart_upload_days")
     expiration_date = parse_date(module.params.get("expiration_date"))
     expiration_days = module.params.get("expiration_days")
     expire_object_delete_marker = module.params.get("expire_object_delete_marker")
+    maximum_object_size = module.params.get("maximum_object_size")
+    minimum_object_size = module.params.get("minimum_object_size")
     noncurrent_version_expiration_days = module.params.get("noncurrent_version_expiration_days")
     noncurrent_version_transition_days = module.params.get("noncurrent_version_transition_days")
     noncurrent_version_transitions = module.params.get("noncurrent_version_transitions")
@@ -292,7 +323,15 @@ def build_rule(client, module):
     transitions = module.params.get("transitions")
     purge_transitions = module.params.get("purge_transitions")
 
-    rule = dict(Filter=dict(Prefix=prefix), Status=status.title())
+    if maximum_object_size is not None or minimum_object_size is not None:
+        and_dict = dict(Prefix=prefix)
+        if minimum_object_size is not None:
+            and_dict["ObjectSizeGreaterThan"] = minimum_object_size
+        if maximum_object_size is not None:
+            and_dict["ObjectSizeLessThan"] = maximum_object_size
+        rule = dict(Filter=dict(And=and_dict), Status=status.title())
+    else:
+        rule = dict(Filter=dict(Prefix=prefix), Status=status.title())
     if rule_id is not None:
         rule["ID"] = rule_id
 
@@ -362,19 +401,16 @@ def compare_and_update_configuration(client, module, current_lifecycle_rules, ru
     lifecycle_configuration = dict(Rules=[])
     changed = False
     appended = False
-
     # If current_lifecycle_obj is not None then we have rules to compare, otherwise just add the rule
     if current_lifecycle_rules:
         # If rule ID exists, use that for comparison otherwise compare based on prefix
         for existing_rule in current_lifecycle_rules:
-            if rule.get("ID") == existing_rule.get("ID") and rule["Filter"].get("Prefix", "") != existing_rule.get(
-                "Filter", {}
-            ).get("Prefix", ""):
-                existing_rule.pop("ID")
-            elif rule_id is None and rule["Filter"].get("Prefix", "") == existing_rule.get("Filter", {}).get(
-                "Prefix", ""
+            if rule.get("ID") == existing_rule.get("ID") and not filters_are_equal(
+                rule.get("Filter"), existing_rule.get("Filter")
             ):
                 existing_rule.pop("ID")
+            elif rule_id is None and filters_are_equal(rule.get("Filter"), existing_rule.get("Filter")):
+                existing_rule.pop("ID")
             if rule.get("ID") == existing_rule.get("ID"):
                 changed_, appended_ = update_or_append_rule(
                     rule, existing_rule, purge_transitions, lifecycle_configuration
@@ -598,6 +634,8 @@ def main():
         expiration_days=dict(type="int"),
         expiration_date=dict(),
         expire_object_delete_marker=dict(type="bool"),
+        maximum_object_size=dict(type="int"),
+        minimum_object_size=dict(type="int"),
         noncurrent_version_expiration_days=dict(type="int"),
         noncurrent_version_keep_newer=dict(type="int"),
         noncurrent_version_storage_class=dict(default="glacier", type="str", choices=s3_storage_class),

tests/integration/targets/s3_lifecycle/tasks/main.yml

@@ -694,6 +694,95 @@
         that:
           - output is changed
 
+    # Check create and delete lifecycle policy with minimum and maximum object size (with prefix)
+    - name: Create rule with minimum and maximum object size
+      s3_lifecycle:
+        name: "{{ bucket_name }}"
+        rule_id: minimum-object-size-prefix
+        prefix: /something/
+        minimum_object_size: 100
+        maximum_object_size: 1000
+        state: present
+        status: enabled
+        expiration_days: 30
+      register: output
+    - assert:
+        that:
+          - output is changed
+
+    - name: Create rule with minimum object size (idempotency)
+      s3_lifecycle:
+        name: "{{ bucket_name }}"
+        rule_id: minimum-object-size-prefix
+        prefix: /something/
+        minimum_object_size: 100
+        maximum_object_size: 1000
+        state: present
+        status: enabled
+        expiration_days: 30
+      register: output
+    - assert:
+        that:
+          - output is not changed
+
+    - name: Delete rule with minimum and maximum object size
+      s3_lifecycle:
+        name: "{{ bucket_name }}"
+        rule_id: minimum-object-size-prefix
+        prefix: /something/
+        minimum_object_size: 100
+        maximum_object_size: 1000
+        state: absent
+        status: enabled
+        expiration_days: 30
+      register: output
+    - assert:
+        that:
+          - output is changed
+
+    # Check create and delete lifecycle policy with minimum and maximum object size (no prefix)
+    - name: Create rule with minimum and maximum object size
+      s3_lifecycle:
+        name: "{{ bucket_name }}"
+        rule_id: minimum-object-size-noprefix
+        minimum_object_size: 100
+        maximum_object_size: 1000
+        state: present
+        status: enabled
+        expiration_days: 30
+      register: output
+    - assert:
+        that:
+          - output is changed
+
+    - name: Create rule with minimum object size (idempotency)
+      s3_lifecycle:
+        name: "{{ bucket_name }}"
+        rule_id: minimum-object-size-noprefix
+        minimum_object_size: 100
+        maximum_object_size: 1000
+        state: present
+        status: enabled
+        expiration_days: 30
+      register: output
+    - assert:
+        that:
+          - output is not changed
+
+    - name: Delete rule with minimum and maximum object size
+      s3_lifecycle:
+        name: "{{ bucket_name }}"
+        rule_id: minimum-object-size-noprefix
+        minimum_object_size: 100
+        maximum_object_size: 1000
+        state: absent
+        status: enabled
+        expiration_days: 30
+      register: output
+    - assert:
+        that:
+          - output is changed
+
  # ============================================================
   always:
     - name: Ensure all buckets are deleted

tests/unit/plugins/modules/s3_lifecycle/test_filters_are_equal.py

@@ -0,0 +1,119 @@
+# -*- coding: utf-8 -*-
+
+# Copyright: (c) 2023, Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+import pytest
+
+from ansible_collections.community.aws.plugins.modules.s3_lifecycle import filters_are_equal
+
+
+@pytest.mark.parametrize(
+    "filter1,filter2,result",
+    [
+        [None, None, True],
+        [{}, {}, True],
+        # Simple filters equal
+        [{"Prefix": ""}, {"Prefix": ""}, True],
+        [{"Prefix": "prefix/"}, {"Prefix": "prefix/"}, True],
+        [{"ObjectSizeGreaterThan": 100}, {"ObjectSizeGreaterThan": 100}, True],
+        [{"ObjectSizeLessThan": 100}, {"ObjectSizeLessThan": 100}, True],
+        # One filter is empty
+        [{"Prefix": ""}, {}, False],
+        [{"ObjectSizeGreaterThan": 100}, {}, False],
+        [{"ObjectSizeLessThan": 100}, {}, False],
+        # One filter is None
+        [{"Prefix": ""}, None, False],
+        [{"ObjectSizeGreaterThan": 100}, None, False],
+        [{"ObjectSizeLessThan": 100}, None, False],
+        # Filters differ in a single key
+        [{"Prefix": "prefix/"}, {"Prefix": "prefix2/"}, False],
+        [{"ObjectSizeGreaterThan": 100}, {"ObjectSizeGreaterThan": 200}, False],
+        [{"ObjectSizeLessThan": 100}, {"ObjectSizeLessThan": 200}, False],
+        # While in theory, these would be equal.  We currently don't treat them as such and
+        # a single key in the "And" dict is technically not valid.
+        [{"Prefix": "prefix/"}, {"And": {"Prefix": "prefix/"}}, False],
+        [{"ObjectSizeGreaterThan": 100}, {"And": {"ObjectSizeGreaterThan": 100}}, False],
+        [{"ObjectSizeLessThan": 100}, {"And": {"ObjectSizeLessThan": 100}}, False],
+    ],
+)
+def test_filters_are_equal_simple(filter1, filter2, result):
+    assert filters_are_equal(filter1, filter2) is result
+    assert filters_are_equal(filter2, filter1) is result  # pylint: disable=arguments-out-of-order
+
+
+# Could be merged with the ones above, but naming will give a better idea of what's wrong
+@pytest.mark.parametrize(
+    "filter1,filter2,result",
+    [
+        # Equal
+        [
+            {"And": {"Prefix": "nested/", "ObjectSizeGreaterThan": 150, "ObjectSizeLessThan": 180}},
+            {"And": {"Prefix": "nested/", "ObjectSizeGreaterThan": 150, "ObjectSizeLessThan": 180}},
+            True,
+        ],
+        # Special case of "Prefix" missing == Prefix of ""
+        [
+            {"And": {"Prefix": "", "ObjectSizeGreaterThan": 150, "ObjectSizeLessThan": 180}},
+            {"And": {"ObjectSizeGreaterThan": 150, "ObjectSizeLessThan": 180}},
+            True,
+        ],
+        # Equal but with 2 of 3 "And" keys
+        [
+            {"And": {"ObjectSizeGreaterThan": 150, "ObjectSizeLessThan": 180}},
+            {"And": {"ObjectSizeGreaterThan": 150, "ObjectSizeLessThan": 180}},
+            True,
+        ],
+        [
+            {"And": {"Prefix": "nested/", "ObjectSizeLessThan": 180}},
+            {"And": {"Prefix": "nested/", "ObjectSizeLessThan": 180}},
+            True,
+        ],
+        [
+            {"And": {"Prefix": "nested/", "ObjectSizeGreaterThan": 150}},
+            {"And": {"Prefix": "nested/", "ObjectSizeGreaterThan": 150}},
+            True,
+        ],
+        # One key missing
+        [
+            {"And": {"Prefix": "nested/", "ObjectSizeGreaterThan": 150, "ObjectSizeLessThan": 180}},
+            {"And": {"ObjectSizeGreaterThan": 150, "ObjectSizeLessThan": 180}},
+            False,
+        ],
+        [
+            {"And": {"Prefix": "nested/", "ObjectSizeGreaterThan": 150, "ObjectSizeLessThan": 180}},
+            {"And": {"Prefix": "nested/", "ObjectSizeLessThan": 180}},
+            False,
+        ],
+        [
+            {"And": {"Prefix": "nested/", "ObjectSizeGreaterThan": 150, "ObjectSizeLessThan": 180}},
+            {"And": {"Prefix": "nested/", "ObjectSizeGreaterThan": 150}},
+            False,
+        ],
+        # One key different
+        [
+            {"And": {"Prefix": "nested/", "ObjectSizeGreaterThan": 150, "ObjectSizeLessThan": 180}},
+            {"And": {"Prefix": "another/", "ObjectSizeGreaterThan": 150, "ObjectSizeLessThan": 180}},
+            False,
+        ],
+        [
+            {"And": {"Prefix": "nested/", "ObjectSizeGreaterThan": 150, "ObjectSizeLessThan": 180}},
+            {"And": {"Prefix": "nested/", "ObjectSizeGreaterThan": 42, "ObjectSizeLessThan": 180}},
+            False,
+        ],
+        [
+            {"And": {"Prefix": "nested/", "ObjectSizeGreaterThan": 150, "ObjectSizeLessThan": 180}},
+            {"And": {"Prefix": "nested/", "ObjectSizeGreaterThan": 150, "ObjectSizeLessThan": 90}},
+            False,
+        ],
+        # Mixed with a non-and
+        [
+            {"Prefix": "test/"},
+            {"And": {"Prefix": "nested/", "ObjectSizeGreaterThan": 150, "ObjectSizeLessThan": 90}},
+            False,
+        ],
+    ],
+)
+def test_filters_are_equal_and(filter1, filter2, result):
+    assert filters_are_equal(filter1, filter2) is result
+    assert filters_are_equal(filter2, filter1) is result  # pylint: disable=arguments-out-of-order