revert back cleanup of all container blobs

ocw/lib/EC2.py

@@ -116,7 +116,7 @@ def cleanup_volumes(self, valid_period_days):
             for volume in response['Volumes']:
                 if EC2.is_outdated(volume['CreateTime'], valid_period_days):
                     if self.volume_protected(volume):
-                        self.log_info('Volume {} has tag DO_NOT_DELETE so protected from deletion',
+                        self.log_info('Volume {} has tag pcw_ignore so protected from deletion',
                                       volume['VolumeId'])
                     elif self.dry_run:
                         self.log_info("Volume deletion of {} skipped due to dry run mode", volume['VolumeId'])
@@ -134,7 +134,7 @@ def cleanup_volumes(self, valid_period_days):
     def volume_protected(self, volume):
         if 'Tags' in volume:
             for tag in volume['Tags']:
-                if tag['Key'] == 'DO_NOT_DELETE':
+                if tag['Key'] == 'pcw_ignore':
                     return True
         return False
 
@@ -199,10 +199,10 @@ def delete_all_clusters(self):
     def cleanup_all(self):
         valid_period_days = PCWConfig.get_feature_property('cleanup', 'ec2-max-age-days', self._namespace)
 
-        if valid_period_days >= 0:
+        if valid_period_days > 0:
             self.cleanup_images(valid_period_days)
-            self.cleanup_volumes(valid_period_days)
             self.cleanup_snapshots(valid_period_days)
+            self.cleanup_volumes(valid_period_days)
         if PCWConfig.getBoolean('cleanup/vpc_cleanup', self._namespace):
             self.cleanup_uploader_vpcs()
 

ocw/lib/azure.py

@@ -6,6 +6,7 @@
 from azure.mgmt.storage import StorageManagementClient
 from azure.storage.blob import BlobServiceClient
 from msrest.exceptions import AuthenticationError
+import re
 import time
 from typing import Dict
 
@@ -102,14 +103,34 @@ def list_by_resource_group(self, resource_group, filters=None):
             resource_group, filter=filters, expand="changedTime")]
 
     def cleanup_all(self):
-        self.cleanup_blob_containers()
         self.cleanup_images_from_rg()
         self.cleanup_disks_from_rg()
+        self.cleanup_blob_containers()
+
+
+    @staticmethod
+    def container_valid_for_cleanup(container):
+        '''
+            under term "container" we meant Azure Blob Storage Container.
+            See https://learn.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction
+            for more details
+            Container is valid for cleanup if it met 2 conditions :
+            1. "metadata" of container does not contain special tag (pcw_ignore)
+            2. Container name or contains "bootdiagnostics-" in its name or its name is "sle-images"
+        '''
+        if 'pcw_ignore' in container['metadata']:
+            return False
+        if re.match('^bootdiagnostics-', container.name):
+            return True
+        if container.name == 'sle-images':
+            return True
+        return False
+
 
     def cleanup_blob_containers(self):
         containers = self.bs_client().list_containers(include_metadata=True)
         for c in containers:
-            if 'pcw_ignore' not in c['metadata']:
+            if Azure.container_valid_for_cleanup(c):
                 self.log_dbg('Found container {}', c.name)
                 container_blobs = self.container_client(c.name).list_blobs()
                 for blob in container_blobs:

tests/test_azure.py

@@ -63,8 +63,11 @@ def __init__(self, managed_by = None):
 
 class FakeBlobContainer:
 
-    def __init__(self, metadata = []):
-        self.name = Faker().uuid4()
+    def __init__(self, metadata = [], name = None):
+        if name is None:
+            self.name = "sle-images"
+        else:
+            self.name = name
         self.metadata = metadata
 
     def __getitem__(self, i):
@@ -269,3 +272,11 @@ def mock_list_resource_groups(self):
     failed_list_resource_groups = 5
     with pytest.raises(AuthenticationError):
         az = Azure('fake')
+
+def test_container_valid_for_cleanup():
+
+    assert Azure.container_valid_for_cleanup(FakeBlobContainer({},"random name")) == False
+    assert Azure.container_valid_for_cleanup(FakeBlobContainer({}, "sle-images")) == True
+    assert Azure.container_valid_for_cleanup(FakeBlobContainer({}, "bootdiagnostics-dsfsdfsdf")) == True
+    assert Azure.container_valid_for_cleanup(FakeBlobContainer({"pcw_ignore": "1"}, "bootdiagnostics-sdafsdfs")) == False
+    assert Azure.container_valid_for_cleanup(FakeBlobContainer({"pcw_ignore": "1"}, "sle-images")) == False

tests/test_ec2.py

@@ -293,7 +293,7 @@ def test_cleanup_volumes_cleanupcheck(ec2_patch):
         'Volumes': [{'VolumeId': MockedEC2Client.volumeid_to_delete, 'CreateTime': older_than_max_age_date},
                     {'VolumeId': 'too_young_to_die', 'CreateTime': now_age_date},
                     {'VolumeId': MockedEC2Client.volumeid_to_delete, 'CreateTime': older_than_max_age_date,
-                     'Tags': [{'Key': 'DO_NOT_DELETE', 'Value': '1'}]}, ]
+                     'Tags': [{'Key': 'pcw_ignore', 'Value': '1'}]}, ]
     }
     ec2_patch.cleanup_volumes(ec2_max_age_days)
     assert len(MockedEC2Client.deleted_volumes) == 1
@@ -442,7 +442,7 @@ def mocked_get_boolean(config_path, field=None):
 
     ec2_patch.cleanup_all()
 
-    assert called_stack == ['cleanup_images', 'cleanup_volumes', 'cleanup_snapshots', 'cleanup_uploader_vpcs']
+    assert called_stack == ['cleanup_images', 'cleanup_snapshots', 'cleanup_volumes', 'cleanup_uploader_vpcs']
 
 def test_list_clusters(ec2_patch, monkeypatch):
     mocked_eks = MockedEKSClient()