From af56b7f09f4d9ba8ad0d2e2ceee0630aaf9d35cd Mon Sep 17 00:00:00 2001 From: Javid Alimohideen Date: Sun, 29 Dec 2024 05:57:14 -0500 Subject: [PATCH] Support min max object size in s3_lifecycle module (#2205) SUMMARY Support the S3 lifecycle settings of minimum and maximum object size to apply the lifecycle rules. Fixes #861 ISSUE TYPE Feature Pull Request COMPONENT NAME s3_lifecycle ADDITIONAL INFORMATION Reviewed-by: Mark Chappell Reviewed-by: Javid Alimohideen --- .../2205-support-minmax-s3lifecycle.yml | 2 + plugins/modules/s3_lifecycle.py | 54 ++++++-- .../targets/s3_lifecycle/tasks/main.yml | 89 +++++++++++++ .../s3_lifecycle/test_filters_are_equal.py | 119 ++++++++++++++++++ 4 files changed, 256 insertions(+), 8 deletions(-) create mode 100644 changelogs/fragments/2205-support-minmax-s3lifecycle.yml create mode 100644 tests/unit/plugins/modules/s3_lifecycle/test_filters_are_equal.py diff --git a/changelogs/fragments/2205-support-minmax-s3lifecycle.yml b/changelogs/fragments/2205-support-minmax-s3lifecycle.yml new file mode 100644 index 00000000000..9eeb11d5f39 --- /dev/null +++ b/changelogs/fragments/2205-support-minmax-s3lifecycle.yml @@ -0,0 +1,2 @@ +minor_changes: + - s3_lifecycle - Support for min and max object size when applying the filter rules (https://github.com/ansible-collections/community.aws/pull/2205). \ No newline at end of file diff --git a/plugins/modules/s3_lifecycle.py b/plugins/modules/s3_lifecycle.py index 2f48e06d404..64bce4516e1 100644 --- a/plugins/modules/s3_lifecycle.py +++ b/plugins/modules/s3_lifecycle.py @@ -59,6 +59,18 @@ and replaced with the new transition(s) default: true type: bool + maximum_object_size: + description: + - The maximum object size to which the rule applies. + required: false + type: int + version_added: 9.1.0 + minimum_object_size: + description: + - The minimum object size to which the rule applies. + required: false + version_added: 9.1.0 + type: int noncurrent_version_expiration_days: description: - The number of days after which non-current versions should be deleted. @@ -272,12 +284,31 @@ def fetch_rules(client, module, name): return current_lifecycle_rules +# Helper function to deeply compare filters +def filters_are_equal(filter1, filter2): + if filter1 == filter2: + return True + if not filter1 or not filter2: + return False + # Treat empty string as equal to a filter not being set + return ( + filter1.get("Prefix", "") == filter2.get("Prefix", "") + and filter1.get("ObjectSizeGreaterThan") == filter2.get("ObjectSizeGreaterThan") + and filter1.get("ObjectSizeLessThan") == filter2.get("ObjectSizeLessThan") + and filter1.get("And", {}).get("Prefix", "") == filter2.get("And", {}).get("Prefix", "") + and filter1.get("And", {}).get("ObjectSizeGreaterThan") == filter2.get("And", {}).get("ObjectSizeGreaterThan") + and filter1.get("And", {}).get("ObjectSizeLessThan") == filter2.get("And", {}).get("ObjectSizeLessThan") + ) + + def build_rule(client, module): name = module.params.get("name") abort_incomplete_multipart_upload_days = module.params.get("abort_incomplete_multipart_upload_days") expiration_date = parse_date(module.params.get("expiration_date")) expiration_days = module.params.get("expiration_days") expire_object_delete_marker = module.params.get("expire_object_delete_marker") + maximum_object_size = module.params.get("maximum_object_size") + minimum_object_size = module.params.get("minimum_object_size") noncurrent_version_expiration_days = module.params.get("noncurrent_version_expiration_days") noncurrent_version_transition_days = module.params.get("noncurrent_version_transition_days") noncurrent_version_transitions = module.params.get("noncurrent_version_transitions") @@ -292,7 +323,15 @@ def build_rule(client, module): transitions = module.params.get("transitions") purge_transitions = module.params.get("purge_transitions") - rule = dict(Filter=dict(Prefix=prefix), Status=status.title()) + if maximum_object_size is not None or minimum_object_size is not None: + and_dict = dict(Prefix=prefix) + if minimum_object_size is not None: + and_dict["ObjectSizeGreaterThan"] = minimum_object_size + if maximum_object_size is not None: + and_dict["ObjectSizeLessThan"] = maximum_object_size + rule = dict(Filter=dict(And=and_dict), Status=status.title()) + else: + rule = dict(Filter=dict(Prefix=prefix), Status=status.title()) if rule_id is not None: rule["ID"] = rule_id @@ -362,19 +401,16 @@ def compare_and_update_configuration(client, module, current_lifecycle_rules, ru lifecycle_configuration = dict(Rules=[]) changed = False appended = False - # If current_lifecycle_obj is not None then we have rules to compare, otherwise just add the rule if current_lifecycle_rules: # If rule ID exists, use that for comparison otherwise compare based on prefix for existing_rule in current_lifecycle_rules: - if rule.get("ID") == existing_rule.get("ID") and rule["Filter"].get("Prefix", "") != existing_rule.get( - "Filter", {} - ).get("Prefix", ""): - existing_rule.pop("ID") - elif rule_id is None and rule["Filter"].get("Prefix", "") == existing_rule.get("Filter", {}).get( - "Prefix", "" + if rule.get("ID") == existing_rule.get("ID") and not filters_are_equal( + rule.get("Filter"), existing_rule.get("Filter") ): existing_rule.pop("ID") + elif rule_id is None and filters_are_equal(rule.get("Filter"), existing_rule.get("Filter")): + existing_rule.pop("ID") if rule.get("ID") == existing_rule.get("ID"): changed_, appended_ = update_or_append_rule( rule, existing_rule, purge_transitions, lifecycle_configuration @@ -598,6 +634,8 @@ def main(): expiration_days=dict(type="int"), expiration_date=dict(), expire_object_delete_marker=dict(type="bool"), + maximum_object_size=dict(type="int"), + minimum_object_size=dict(type="int"), noncurrent_version_expiration_days=dict(type="int"), noncurrent_version_keep_newer=dict(type="int"), noncurrent_version_storage_class=dict(default="glacier", type="str", choices=s3_storage_class), diff --git a/tests/integration/targets/s3_lifecycle/tasks/main.yml b/tests/integration/targets/s3_lifecycle/tasks/main.yml index d9f169561af..947c356ef47 100644 --- a/tests/integration/targets/s3_lifecycle/tasks/main.yml +++ b/tests/integration/targets/s3_lifecycle/tasks/main.yml @@ -694,6 +694,95 @@ that: - output is changed + # Check create and delete lifecycle policy with minimum and maximum object size (with prefix) + - name: Create rule with minimum and maximum object size + s3_lifecycle: + name: "{{ bucket_name }}" + rule_id: minimum-object-size-prefix + prefix: /something/ + minimum_object_size: 100 + maximum_object_size: 1000 + state: present + status: enabled + expiration_days: 30 + register: output + - assert: + that: + - output is changed + + - name: Create rule with minimum object size (idempotency) + s3_lifecycle: + name: "{{ bucket_name }}" + rule_id: minimum-object-size-prefix + prefix: /something/ + minimum_object_size: 100 + maximum_object_size: 1000 + state: present + status: enabled + expiration_days: 30 + register: output + - assert: + that: + - output is not changed + + - name: Delete rule with minimum and maximum object size + s3_lifecycle: + name: "{{ bucket_name }}" + rule_id: minimum-object-size-prefix + prefix: /something/ + minimum_object_size: 100 + maximum_object_size: 1000 + state: absent + status: enabled + expiration_days: 30 + register: output + - assert: + that: + - output is changed + + # Check create and delete lifecycle policy with minimum and maximum object size (no prefix) + - name: Create rule with minimum and maximum object size + s3_lifecycle: + name: "{{ bucket_name }}" + rule_id: minimum-object-size-noprefix + minimum_object_size: 100 + maximum_object_size: 1000 + state: present + status: enabled + expiration_days: 30 + register: output + - assert: + that: + - output is changed + + - name: Create rule with minimum object size (idempotency) + s3_lifecycle: + name: "{{ bucket_name }}" + rule_id: minimum-object-size-noprefix + minimum_object_size: 100 + maximum_object_size: 1000 + state: present + status: enabled + expiration_days: 30 + register: output + - assert: + that: + - output is not changed + + - name: Delete rule with minimum and maximum object size + s3_lifecycle: + name: "{{ bucket_name }}" + rule_id: minimum-object-size-noprefix + minimum_object_size: 100 + maximum_object_size: 1000 + state: absent + status: enabled + expiration_days: 30 + register: output + - assert: + that: + - output is changed + # ============================================================ always: - name: Ensure all buckets are deleted diff --git a/tests/unit/plugins/modules/s3_lifecycle/test_filters_are_equal.py b/tests/unit/plugins/modules/s3_lifecycle/test_filters_are_equal.py new file mode 100644 index 00000000000..4f295646300 --- /dev/null +++ b/tests/unit/plugins/modules/s3_lifecycle/test_filters_are_equal.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- + +# Copyright: (c) 2023, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +import pytest + +from ansible_collections.community.aws.plugins.modules.s3_lifecycle import filters_are_equal + + +@pytest.mark.parametrize( + "filter1,filter2,result", + [ + [None, None, True], + [{}, {}, True], + # Simple filters equal + [{"Prefix": ""}, {"Prefix": ""}, True], + [{"Prefix": "prefix/"}, {"Prefix": "prefix/"}, True], + [{"ObjectSizeGreaterThan": 100}, {"ObjectSizeGreaterThan": 100}, True], + [{"ObjectSizeLessThan": 100}, {"ObjectSizeLessThan": 100}, True], + # One filter is empty + [{"Prefix": ""}, {}, False], + [{"ObjectSizeGreaterThan": 100}, {}, False], + [{"ObjectSizeLessThan": 100}, {}, False], + # One filter is None + [{"Prefix": ""}, None, False], + [{"ObjectSizeGreaterThan": 100}, None, False], + [{"ObjectSizeLessThan": 100}, None, False], + # Filters differ in a single key + [{"Prefix": "prefix/"}, {"Prefix": "prefix2/"}, False], + [{"ObjectSizeGreaterThan": 100}, {"ObjectSizeGreaterThan": 200}, False], + [{"ObjectSizeLessThan": 100}, {"ObjectSizeLessThan": 200}, False], + # While in theory, these would be equal. We currently don't treat them as such and + # a single key in the "And" dict is technically not valid. + [{"Prefix": "prefix/"}, {"And": {"Prefix": "prefix/"}}, False], + [{"ObjectSizeGreaterThan": 100}, {"And": {"ObjectSizeGreaterThan": 100}}, False], + [{"ObjectSizeLessThan": 100}, {"And": {"ObjectSizeLessThan": 100}}, False], + ], +) +def test_filters_are_equal_simple(filter1, filter2, result): + assert filters_are_equal(filter1, filter2) is result + assert filters_are_equal(filter2, filter1) is result # pylint: disable=arguments-out-of-order + + +# Could be merged with the ones above, but naming will give a better idea of what's wrong +@pytest.mark.parametrize( + "filter1,filter2,result", + [ + # Equal + [ + {"And": {"Prefix": "nested/", "ObjectSizeGreaterThan": 150, "ObjectSizeLessThan": 180}}, + {"And": {"Prefix": "nested/", "ObjectSizeGreaterThan": 150, "ObjectSizeLessThan": 180}}, + True, + ], + # Special case of "Prefix" missing == Prefix of "" + [ + {"And": {"Prefix": "", "ObjectSizeGreaterThan": 150, "ObjectSizeLessThan": 180}}, + {"And": {"ObjectSizeGreaterThan": 150, "ObjectSizeLessThan": 180}}, + True, + ], + # Equal but with 2 of 3 "And" keys + [ + {"And": {"ObjectSizeGreaterThan": 150, "ObjectSizeLessThan": 180}}, + {"And": {"ObjectSizeGreaterThan": 150, "ObjectSizeLessThan": 180}}, + True, + ], + [ + {"And": {"Prefix": "nested/", "ObjectSizeLessThan": 180}}, + {"And": {"Prefix": "nested/", "ObjectSizeLessThan": 180}}, + True, + ], + [ + {"And": {"Prefix": "nested/", "ObjectSizeGreaterThan": 150}}, + {"And": {"Prefix": "nested/", "ObjectSizeGreaterThan": 150}}, + True, + ], + # One key missing + [ + {"And": {"Prefix": "nested/", "ObjectSizeGreaterThan": 150, "ObjectSizeLessThan": 180}}, + {"And": {"ObjectSizeGreaterThan": 150, "ObjectSizeLessThan": 180}}, + False, + ], + [ + {"And": {"Prefix": "nested/", "ObjectSizeGreaterThan": 150, "ObjectSizeLessThan": 180}}, + {"And": {"Prefix": "nested/", "ObjectSizeLessThan": 180}}, + False, + ], + [ + {"And": {"Prefix": "nested/", "ObjectSizeGreaterThan": 150, "ObjectSizeLessThan": 180}}, + {"And": {"Prefix": "nested/", "ObjectSizeGreaterThan": 150}}, + False, + ], + # One key different + [ + {"And": {"Prefix": "nested/", "ObjectSizeGreaterThan": 150, "ObjectSizeLessThan": 180}}, + {"And": {"Prefix": "another/", "ObjectSizeGreaterThan": 150, "ObjectSizeLessThan": 180}}, + False, + ], + [ + {"And": {"Prefix": "nested/", "ObjectSizeGreaterThan": 150, "ObjectSizeLessThan": 180}}, + {"And": {"Prefix": "nested/", "ObjectSizeGreaterThan": 42, "ObjectSizeLessThan": 180}}, + False, + ], + [ + {"And": {"Prefix": "nested/", "ObjectSizeGreaterThan": 150, "ObjectSizeLessThan": 180}}, + {"And": {"Prefix": "nested/", "ObjectSizeGreaterThan": 150, "ObjectSizeLessThan": 90}}, + False, + ], + # Mixed with a non-and + [ + {"Prefix": "test/"}, + {"And": {"Prefix": "nested/", "ObjectSizeGreaterThan": 150, "ObjectSizeLessThan": 90}}, + False, + ], + ], +) +def test_filters_are_equal_and(filter1, filter2, result): + assert filters_are_equal(filter1, filter2) is result + assert filters_are_equal(filter2, filter1) is result # pylint: disable=arguments-out-of-order