azure: Add support for cleaning up Image Galleries

ocw/lib/azure.py

@@ -7,6 +7,7 @@
 from azure.storage.blob import BlobServiceClient
 from azure.core.exceptions import ResourceNotFoundError
 from msrest.exceptions import AuthenticationError
+from dateutil.parser import parse
 from webui.PCWConfig import PCWConfig
 from .provider import Provider
 from ..models import Instance
@@ -19,6 +20,7 @@ def __init__(self, namespace: str):
         super().__init__(namespace)
         self.__resource_group = PCWConfig.get_feature_property('cleanup', 'azure-storage-resourcegroup', namespace)
         self.check_credentials()
+        self.__gallery = PCWConfig.get_feature_property('cleanup', 'azure-gallery-name', namespace)
 
     def __new__(cls, vault_namespace: str) -> 'Azure':
         if vault_namespace not in Azure.__instances:
@@ -96,6 +98,9 @@ def get_vm_types_in_resource_group(self, resource_group: str) -> str:
             return ', '.join(type_set)
         return "N/A"
 
+    def get_resource_properties(self, resource_id):
+        return self.resource_mgmt_client().resources.get_by_id(resource_id, api_version="2023-07-03").properties
+
     def list_resource_groups(self) -> list:
         return list(self.resource_mgmt_client().resource_groups.list())
 
@@ -106,22 +111,21 @@ def delete_resource(self, resource_id: str) -> None:
             self.log_info(f"Deleting of resource group {resource_id}")
             self.resource_mgmt_client().resource_groups.begin_delete(resource_id)
 
-    def list_images_by_resource_group(self, resource_group):
-        return self.list_by_resource_group(resource_group,
-                                           filters="resourceType eq 'Microsoft.Compute/images'")
+    def list_images(self):
+        return self.list_resource(filters="resourceType eq 'Microsoft.Compute/images'")
 
-    def list_disks_by_resource_group(self, resource_group):
-        return self.list_by_resource_group(resource_group,
-                                           filters="resourceType eq 'Microsoft.Compute/disks'")
+    def list_disks(self):
+        return self.list_resource(filters="resourceType eq 'Microsoft.Compute/disks'")
 
-    def list_by_resource_group(self, resource_group, filters=None) -> list:
+    def list_resource(self, filters=None) -> list:
         return list(self.resource_mgmt_client().resources.list_by_resource_group(
-            resource_group, filter=filters, expand="changedTime"))
+            self.__resource_group, filter=filters, expand="changedTime"))
 
     def cleanup_all(self) -> None:
         self.log_info("Call cleanup_all")
-        self.cleanup_images_from_rg()
-        self.cleanup_disks_from_rg()
+        self.cleanup_images()
+        self.cleanup_gallery_img_versions()
+        self.cleanup_disks()
         self.cleanup_blob_containers()
 
     @staticmethod
@@ -157,25 +161,50 @@ def cleanup_blob_containers(self) -> None:
                             self.log_info(f"Deleting blob {blob.name}")
                             self.container_client(container.name).delete_blob(blob.name, delete_snapshots="include")
 
-    def cleanup_images_from_rg(self) -> None:
-        self.log_dbg("Call cleanup_images_from_rg")
-        for item in self.list_images_by_resource_group(self.__resource_group):
+    def cleanup_images(self) -> None:
+        self.log_dbg("Call cleanup_images")
+        for item in self.list_images():
             if self.is_outdated(item.changed_time):
                 if self.dry_run:
                     self.log_info(f"Deletion of image {item.name} skipped due to dry run mode")
                 else:
                     self.log_info(f"Delete image '{item.name}'")
                     self.compute_mgmt_client().images.begin_delete(self.__resource_group, item.name)
 
-    def cleanup_disks_from_rg(self) -> None:
-        self.log_dbg("Call cleanup_disks_from_rg")
-        for item in self.list_disks_by_resource_group(self.__resource_group):
+    def cleanup_disks(self) -> None:
+        self.log_dbg("Call cleanup_disks")
+        for item in self.list_disks():
             if self.is_outdated(item.changed_time):
                 if self.compute_mgmt_client().disks.get(self.__resource_group, item.name).managed_by:
-                    self.log_warn(f"Disk is in use - unable delete {item.name}")
+                    self.log_warn(f"Disk is in use - skipping {item.name}")
                 else:
                     if self.dry_run:
                         self.log_info(f"Deletion of disk {item.name} skipped due to dry run mode")
                     else:
                         self.log_info(f"Delete disk '{item.name}'")
                         self.compute_mgmt_client().disks.begin_delete(self.__resource_group, item.name)
+
+    def cleanup_gallery_img_versions(self) -> None:
+        self.log_dbg("Call cleanup_gallery_img_versions")
+        gallery = self.compute_mgmt_client().galleries.get(self.__resource_group, self.__gallery)
+        if Instance.TAG_IGNORE in gallery.tags:
+            self.log_err(f"Gallery in resource group {self.__resource_group} has {Instance.TAG_IGNORE} tag: {self.__gallery}")
+            return
+        for image in self.compute_mgmt_client().gallery_images.list_by_gallery(self.__resource_group, gallery.name):
+            if Instance.TAG_IGNORE in image.tags:
+                self.log_info(f"Gallery {self.__gallery} image {image} has {Instance.TAG_IGNORE} tag")
+                continue
+            for version in self.compute_mgmt_client().gallery_image_versions.list_by_gallery_image(
+                    self.__resource_group, gallery.name, image.name):
+                if Instance.TAG_IGNORE in version.tags:
+                    self.log_info(f"Image version {version} for image {image} in gallery {self.__gallery} has {Instance.TAG_IGNORE} tag")
+                    continue
+                properties = self.get_resource_properties(version.id)
+                if self.is_outdated(parse(properties['publishingProfile']['publishedDate'])):
+                    if self.dry_run:
+                        self.log_info(f"Deletion of version {gallery.name}/{image.name}/{version.name} skipped due to dry run mode")
+                    else:
+                        self.log_info(f"Delete version '{gallery.name}/{image.name}/{version.name}'")
+                        self.compute_mgmt_client().gallery_image_versions.begin_delete(
+                                self.__resource_group, gallery.name, image.name, version.name
+                        )

tests/test_azure.py

@@ -160,29 +160,27 @@ def test_cleanup_blob_containers_all_new_one_pcw_ignore(azure_patch, container_c
     assert container_client_all_new.deleted_blobs == []
 
 
-def test_cleanup_images_from_rg_all_new(azure_patch, monkeypatch, mock_compute_mgmt_client):
-    monkeypatch.setattr(Azure, 'list_by_resource_group', lambda *args, **kwargs: [FakeItem(), FakeItem()])
-    azure_patch.cleanup_images_from_rg()
+def test_cleanup_images_all_new(azure_patch, monkeypatch, mock_compute_mgmt_client):
+    monkeypatch.setattr(Azure, 'list_resource', lambda *args, **kwargs: [FakeItem(), FakeItem()])
+    azure_patch.cleanup_images()
 
     assert len(deleted_images) == 0
 
 
-def test_cleanup_images_from_rg_one_old(azure_patch, monkeypatch, mock_compute_mgmt_client):
+def test_cleanup_images_one_old(azure_patch, monkeypatch, mock_compute_mgmt_client):
     old_times = datetime.now(timezone.utc) - timedelta(hours=generators.max_age_hours+1)
-    monkeypatch.setattr(Azure, 'list_by_resource_group', lambda *args, **kwargs: [FakeItem(old_times, "to_delete"),
-                                                                                  FakeItem()
-                                                                                  ])
+    monkeypatch.setattr(Azure, 'list_resource', lambda *args, **kwargs: [FakeItem(old_times, "to_delete"), FakeItem()])
     azure_patch.dry_run = True
-    azure_patch.cleanup_images_from_rg()
+    azure_patch.cleanup_images()
     assert len(deleted_images) == 0
 
     azure_patch.dry_run = False
-    azure_patch.cleanup_images_from_rg()
+    azure_patch.cleanup_images()
     assert len(deleted_images) == 1
     assert deleted_images[0] == "to_delete"
 
 
-def test_cleanup_disks_from_rg_all_new(azure_patch, monkeypatch):
+def test_cleanup_disks_all_new(azure_patch, monkeypatch):
 
     global deleted_images
     # to make sure that we not failing due to other test left dirty env.
@@ -199,13 +197,13 @@ def compute_mgmt_client():
 
     monkeypatch.setattr(Azure, 'compute_mgmt_client', mock_compute_mgmt_client)
 
-    monkeypatch.setattr(Azure, 'list_by_resource_group', lambda *args, **kwargs: [FakeItem(), FakeItem()])
-    azure_patch.cleanup_disks_from_rg()
+    monkeypatch.setattr(Azure, 'list_resource', lambda *args, **kwargs: [FakeItem(), FakeItem()])
+    azure_patch.cleanup_disks()
 
     assert len(deleted_images) == 0
 
 
-def test_cleanup_disks_from_rg_one_old_no_managed_by(azure_patch, monkeypatch):
+def test_cleanup_disks_one_old_no_managed_by(azure_patch, monkeypatch):
     global deleted_images
     # to make sure that we not failing due to other test left dirty env.
     deleted_images = list()
@@ -222,20 +220,18 @@ def compute_mgmt_client():
     monkeypatch.setattr(Azure, 'compute_mgmt_client', mock_compute_mgmt_client)
 
     old_times = datetime.now(timezone.utc) - timedelta(hours=generators.max_age_hours+1)
-    monkeypatch.setattr(Azure, 'list_by_resource_group', lambda *args, **kwargs: [FakeItem(old_times, "to_delete"),
-                                                                                  FakeItem()
-                                                                                  ])
+    monkeypatch.setattr(Azure, 'list_resource', lambda *args, **kwargs: [FakeItem(old_times, "to_delete"), FakeItem()])
     azure_patch.dry_run = True
-    azure_patch.cleanup_disks_from_rg()
+    azure_patch.cleanup_disks()
     assert len(deleted_images) == 0
 
     azure_patch.dry_run = False
-    azure_patch.cleanup_disks_from_rg()
+    azure_patch.cleanup_disks()
     assert len(deleted_images) == 1
     assert deleted_images[0] == "to_delete"
 
 
-def test_cleanup_disks_from_rg_one_old_with_managed_by(azure_patch, monkeypatch):
+def test_cleanup_disks_one_old_with_managed_by(azure_patch, monkeypatch):
     global deleted_images
     # to make sure that we not failing due to other test left dirty env.
     deleted_images = list()
@@ -252,10 +248,8 @@ def compute_mgmt_client():
     monkeypatch.setattr(Azure, 'compute_mgmt_client', mock_compute_mgmt_client)
 
     old_times = datetime.now(timezone.utc) - timedelta(hours=generators.max_age_hours+1)
-    monkeypatch.setattr(Azure, 'list_by_resource_group', lambda *args, **kwargs: [FakeItem(old_times, "to_delete"),
-                                                                                  FakeItem()
-                                                                                  ])
-    azure_patch.cleanup_disks_from_rg()
+    monkeypatch.setattr(Azure, 'list_resource', lambda *args, **kwargs: [FakeItem(old_times, "to_delete"), FakeItem()])
+    azure_patch.cleanup_disks()
 
     assert len(deleted_images) == 0
 
@@ -269,13 +263,14 @@ def count_call(*args, **kwargs):
 
     monkeypatch.setattr(Azure, 'get_storage_key', lambda *args, **kwargs: 'FOOXX')
     monkeypatch.setattr(Azure, 'cleanup_blob_containers', count_call)
-    monkeypatch.setattr(Azure, 'cleanup_disks_from_rg', count_call)
-    monkeypatch.setattr(Azure, 'cleanup_images_from_rg', count_call)
+    monkeypatch.setattr(Azure, 'cleanup_disks', count_call)
+    monkeypatch.setattr(Azure, 'cleanup_images', count_call)
+    monkeypatch.setattr(Azure, 'cleanup_gallery_img_versions', count_call)
     monkeypatch.setattr(Provider, 'read_auth_json', lambda *args, **kwargs: '{}')
 
     az = Azure('fake')
     az.cleanup_all()
-    assert called == 3
+    assert called == 4
 
 
 def test_check_credentials(monkeypatch):

webui/PCWConfig.py

@@ -56,6 +56,7 @@ class PCWConfig():
     def get_feature_property(feature: str, property: str, namespace: str = None):
         default_values = {
             'cleanup/max-age-hours': {'default': 24 * 7, 'return_type': int},
+            'cleanup/azure-gallery-name': {'default': 'test_image_gallery', 'return_type': str},
             'cleanup/azure-storage-resourcegroup': {'default': 'openqa-upload', 'return_type': str},
             'cleanup/azure-storage-account-name': {'default': 'openqa', 'return_type': str},
             'cleanup/ec2-max-age-days': {'default': -1, 'return_type': int},